X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvnet%2Fdevices%2Fvirtio%2Fvhost_user_input.c;h=a371e57e8e38b7e60a10f8efede3bf1b1656b475;hb=b232d192f2f77114e26cf7f99b57bcd6140872d0;hp=4fdfbd5d6e0d7b313c2b33e38174122d4ac06855;hpb=9282538f0984590369d8096704de61301889d8b0;p=vpp.git

diff --git a/src/vnet/devices/virtio/vhost_user_input.c b/src/vnet/devices/virtio/vhost_user_input.c
index 4fdfbd5d6e0..a371e57e8e3 100644
--- a/src/vnet/devices/virtio/vhost_user_input.c
+++ b/src/vnet/devices/virtio/vhost_user_input.c
@@ -39,6 +39,7 @@
 #include <vnet/devices/devices.h>
 #include <vnet/feature/feature.h>
 
+#include <vnet/devices/virtio/virtio.h>
 #include <vnet/devices/virtio/vhost_user.h>
 #include <vnet/devices/virtio/vhost_user_inline.h>
 
@@ -65,7 +66,7 @@
  */
 #define VHOST_USER_RX_COPY_THRESHOLD 64
 
-vlib_node_registration_t vhost_user_input_node;
+extern vlib_node_registration_t vhost_user_input_node;
 
 #define foreach_vhost_user_input_func_error      \
   _(NO_ERROR, "no error")  \
@@ -92,10 +93,10 @@ static __clib_unused char *vhost_user_input_func_error_strings[] = {
 static_always_inline void
 vhost_user_rx_trace (vhost_trace_t * t,
 		     vhost_user_intf_t * vui, u16 qid,
-		     vlib_buffer_t * b, vhost_user_vring_t * txvq)
+		     vlib_buffer_t * b, vhost_user_vring_t * txvq,
+		     u16 last_avail_idx)
 {
   vhost_user_main_t *vum = &vhost_user_main;
-  u32 last_avail_idx = txvq->last_avail_idx;
   u32 desc_current = txvq->avail->ring[last_avail_idx & txvq->qsz_mask];
   vring_desc_t *hdr_desc = 0;
   virtio_net_hdr_mrg_rxbuf_t *hdr;
@@ -195,25 +196,27 @@ vhost_user_rx_discard_packet (vlib_main_t * vm,
    */
   u32 discarded_packets = 0;
   u32 avail_idx = txvq->avail->idx;
+  u16 mask = txvq->qsz_mask;
+  u16 last_avail_idx = txvq->last_avail_idx;
+  u16 last_used_idx = txvq->last_used_idx;
   while (discarded_packets != discard_max)
     {
-      if (avail_idx == txvq->last_avail_idx)
+      if (avail_idx == last_avail_idx)
 	goto out;
 
-      u16 desc_chain_head =
-	txvq->avail->ring[txvq->last_avail_idx & txvq->qsz_mask];
-      txvq->last_avail_idx++;
-      txvq->used->ring[txvq->last_used_idx & txvq->qsz_mask].id =
-	desc_chain_head;
-      txvq->used->ring[txvq->last_used_idx & txvq->qsz_mask].len = 0;
-      vhost_user_log_dirty_ring (vui, txvq,
-				 ring[txvq->last_used_idx & txvq->qsz_mask]);
-      txvq->last_used_idx++;
+      u16 desc_chain_head = txvq->avail->ring[last_avail_idx & mask];
+      last_avail_idx++;
+      txvq->used->ring[last_used_idx & mask].id = desc_chain_head;
+      txvq->used->ring[last_used_idx & mask].len = 0;
+      vhost_user_log_dirty_ring (vui, txvq, ring[last_used_idx & mask]);
+      last_used_idx++;
       discarded_packets++;
     }
 
 out:
-  CLIB_MEMORY_BARRIER ();
+  txvq->last_avail_idx = last_avail_idx;
+  txvq->last_used_idx = last_used_idx;
+  CLIB_MEMORY_STORE_BARRIER ();
   txvq->used->idx = txvq->last_used_idx;
   vhost_user_log_dirty_ring (vui, txvq, idx);
   return discarded_packets;
@@ -222,7 +225,7 @@ out:
 /*
  * In case of overflow, we need to rewind the array of allocated buffers.
  */
-static __clib_unused void
+static_always_inline void
 vhost_user_input_rewind_buffers (vlib_main_t * vm,
 				 vhost_cpu_t * cpu, vlib_buffer_t * b_head)
 {
@@ -241,12 +244,97 @@ vhost_user_input_rewind_buffers (vlib_main_t * vm,
   cpu->rx_buffers_len++;
 }
 
-static __clib_unused u32
+static_always_inline void
+vhost_user_handle_rx_offload (vlib_buffer_t * b0, u8 * b0_data,
+			      virtio_net_hdr_t * hdr)
+{
+  u8 l4_hdr_sz = 0;
+
+  if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
+    {
+      u8 l4_proto = 0;
+      ethernet_header_t *eh = (ethernet_header_t *) b0_data;
+      u16 ethertype = clib_net_to_host_u16 (eh->type);
+      u16 l2hdr_sz = sizeof (ethernet_header_t);
+
+      if (ethernet_frame_is_tagged (ethertype))
+	{
+	  ethernet_vlan_header_t *vlan = (ethernet_vlan_header_t *) (eh + 1);
+
+	  ethertype = clib_net_to_host_u16 (vlan->type);
+	  l2hdr_sz += sizeof (*vlan);
+	  if (ethertype == ETHERNET_TYPE_VLAN)
+	    {
+	      vlan++;
+	      ethertype = clib_net_to_host_u16 (vlan->type);
+	      l2hdr_sz += sizeof (*vlan);
+	    }
+	}
+      vnet_buffer (b0)->l2_hdr_offset = 0;
+      vnet_buffer (b0)->l3_hdr_offset = l2hdr_sz;
+      vnet_buffer (b0)->l4_hdr_offset = hdr->csum_start;
+      b0->flags |= (VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
+		    VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
+		    VNET_BUFFER_F_L4_HDR_OFFSET_VALID |
+		    VNET_BUFFER_F_OFFLOAD_IP_CKSUM);
+
+      if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP4))
+	{
+	  ip4_header_t *ip4 = (ip4_header_t *) (b0_data + l2hdr_sz);
+	  l4_proto = ip4->protocol;
+	  b0->flags |= VNET_BUFFER_F_IS_IP4;
+	}
+      else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6))
+	{
+	  ip6_header_t *ip6 = (ip6_header_t *) (b0_data + l2hdr_sz);
+	  l4_proto = ip6->protocol;
+	  b0->flags |= VNET_BUFFER_F_IS_IP6;
+	}
+
+      if (l4_proto == IP_PROTOCOL_TCP)
+	{
+	  tcp_header_t *tcp = (tcp_header_t *)
+	    (b0_data + vnet_buffer (b0)->l4_hdr_offset);
+	  l4_hdr_sz = tcp_header_bytes (tcp);
+	  tcp->checksum = 0;
+	  b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+	}
+      else if (l4_proto == IP_PROTOCOL_UDP)
+	{
+	  udp_header_t *udp =
+	    (udp_header_t *) (b0_data + vnet_buffer (b0)->l4_hdr_offset);
+	  l4_hdr_sz = sizeof (*udp);
+	  udp->checksum = 0;
+	  b0->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+	}
+    }
+
+  if (hdr->gso_type == VIRTIO_NET_HDR_GSO_UDP)
+    {
+      vnet_buffer2 (b0)->gso_size = hdr->gso_size;
+      vnet_buffer2 (b0)->gso_l4_hdr_sz = l4_hdr_sz;
+      b0->flags |= VNET_BUFFER_F_GSO;
+    }
+  else if (hdr->gso_type == VIRTIO_NET_HDR_GSO_TCPV4)
+    {
+      vnet_buffer2 (b0)->gso_size = hdr->gso_size;
+      vnet_buffer2 (b0)->gso_l4_hdr_sz = l4_hdr_sz;
+      b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP4);
+    }
+  else if (hdr->gso_type == VIRTIO_NET_HDR_GSO_TCPV6)
+    {
+      vnet_buffer2 (b0)->gso_size = hdr->gso_size;
+      vnet_buffer2 (b0)->gso_l4_hdr_sz = l4_hdr_sz;
+      b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP6);
+    }
+}
+
+static_always_inline u32
 vhost_user_if_input (vlib_main_t * vm,
 		     vhost_user_main_t * vum,
 		     vhost_user_intf_t * vui,
 		     u16 qid, vlib_node_runtime_t * node,
-		     vnet_hw_interface_rx_mode mode)
+		     vnet_hw_interface_rx_mode mode, u8 enable_csum)
 {
   vhost_user_vring_t *txvq = &vui->vrings[VHOST_VRING_IDX_TX (qid)];
   vnet_feature_main_t *fm = &feature_main;
@@ -256,15 +344,17 @@ vhost_user_if_input (vlib_main_t * vm,
   u32 n_left_to_next, *to_next;
   u32 next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
   u32 n_trace = vlib_get_trace_count (vm, node);
+  u32 buffer_data_size = vlib_buffer_get_default_data_size (vm);
   u32 map_hint = 0;
   vhost_cpu_t *cpu = &vum->cpus[vm->thread_index];
   u16 copy_len = 0;
   u8 feature_arc_idx = fm->device_input_feature_arc_index;
   u32 current_config_index = ~(u32) 0;
+  u16 mask = txvq->qsz_mask;
 
   /* The descriptor table is not ready yet */
   if (PREDICT_FALSE (txvq->avail == 0))
-    return 0;
+    goto done;
 
   {
     /* do we have pending interrupts ? */
@@ -299,13 +389,13 @@ vhost_user_if_input (vlib_main_t * vm,
     }
 
   if (PREDICT_FALSE (txvq->avail->flags & 0xFFFE))
-    return 0;
+    goto done;
 
   n_left = (u16) (txvq->avail->idx - txvq->last_avail_idx);
 
   /* nothing to do */
   if (PREDICT_FALSE (n_left == 0))
-    return 0;
+    goto done;
 
   if (PREDICT_FALSE (!vui->admin_up || !(txvq->enabled)))
     {
@@ -318,10 +408,10 @@ vhost_user_if_input (vlib_main_t * vm,
        */
       vhost_user_rx_discard_packet (vm, vui, txvq,
 				    VHOST_USER_DOWN_DISCARD_COUNT);
-      return 0;
+      goto done;
     }
 
-  if (PREDICT_FALSE (n_left == (txvq->qsz_mask + 1)))
+  if (PREDICT_FALSE (n_left == (mask + 1)))
     {
       /*
        * Informational error logging when VPP is not
@@ -336,7 +426,7 @@ vhost_user_if_input (vlib_main_t * vm,
 
   /*
    * For small packets (<2kB), we will not need more than one vlib buffer
-   * per packet. In case packets are bigger, we will just yeld at some point
+   * per packet. In case packets are bigger, we will just yield at some point
    * in the loop and come back later. This is not an issue as for big packet,
    * processing cost really comes from the memory copy.
    * The assumption is that big packets will fit in 40 buffers.
@@ -346,9 +436,8 @@ vhost_user_if_input (vlib_main_t * vm,
     {
       u32 curr_len = cpu->rx_buffers_len;
       cpu->rx_buffers_len +=
-	vlib_buffer_alloc_from_free_list (vm, cpu->rx_buffers + curr_len,
-					  VHOST_USER_RX_BUFFERS_N - curr_len,
-					  VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
+	vlib_buffer_alloc (vm, cpu->rx_buffers + curr_len,
+			   VHOST_USER_RX_BUFFERS_N - curr_len);
 
       if (PREDICT_FALSE
 	  (cpu->rx_buffers_len < VHOST_USER_RX_BUFFER_STARVATION))
@@ -382,6 +471,9 @@ vhost_user_if_input (vlib_main_t * vm,
 			    &next_index, 0);
     }
 
+  u16 last_avail_idx = txvq->last_avail_idx;
+  u16 last_used_idx = txvq->last_used_idx;
+
   vlib_get_new_next_frame (vm, node, next_index, to_next, n_left_to_next);
 
   if (next_index == VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT)
@@ -391,12 +483,13 @@ vhost_user_if_input (vlib_main_t * vm,
       vlib_frame_t *f;
       ethernet_input_frame_t *ef;
       nf = vlib_node_runtime_get_next_frame (vm, node, next_index);
-      f = vlib_get_frame (vm, nf->frame_index);
+      f = vlib_get_frame (vm, nf->frame);
       f->flags = ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX;
 
       ef = vlib_frame_scalar_args (f);
       ef->sw_if_index = vui->sw_if_index;
       ef->hw_if_index = vui->hw_if_index;
+      vlib_frame_no_append (f);
     }
 
   while (n_left > 0)
@@ -417,7 +510,7 @@ vhost_user_if_input (vlib_main_t * vm,
 	  break;
 	}
 
-      desc_current = txvq->avail->ring[txvq->last_avail_idx & txvq->qsz_mask];
+      desc_current = txvq->avail->ring[last_avail_idx & mask];
       cpu->rx_buffers_len--;
       bi_current = cpu->rx_buffers[cpu->rx_buffers_len];
       b_head = b_current = vlib_get_buffer (vm, bi_current);
@@ -429,11 +522,9 @@ vhost_user_if_input (vlib_main_t * vm,
 	(vm, cpu->rx_buffers[cpu->rx_buffers_len - 1], LOAD);
 
       /* Just preset the used descriptor id and length for later */
-      txvq->used->ring[txvq->last_used_idx & txvq->qsz_mask].id =
-	desc_current;
-      txvq->used->ring[txvq->last_used_idx & txvq->qsz_mask].len = 0;
-      vhost_user_log_dirty_ring (vui, txvq,
-				 ring[txvq->last_used_idx & txvq->qsz_mask]);
+      txvq->used->ring[last_used_idx & mask].id = desc_current;
+      txvq->used->ring[last_used_idx & mask].len = 0;
+      vhost_user_log_dirty_ring (vui, txvq, ring[last_used_idx & mask]);
 
       /* The buffer should already be initialized */
       b_head->total_length_not_including_first_buffer = 0;
@@ -441,12 +532,11 @@ vhost_user_if_input (vlib_main_t * vm,
 
       if (PREDICT_FALSE (n_trace))
 	{
-	  //TODO: next_index is not exactly known at that point
 	  vlib_trace_buffer (vm, node, next_index, b_head,
 			     /* follow_chain */ 0);
 	  vhost_trace_t *t0 =
 	    vlib_add_trace (vm, node, b_head, sizeof (t0[0]));
-	  vhost_user_rx_trace (t0, vui, qid, b_head, txvq);
+	  vhost_user_rx_trace (t0, vui, qid, b_head, txvq, last_avail_idx);
 	  n_trace--;
 	  vlib_set_trace_count (vm, node, n_trace);
 	}
@@ -467,16 +557,42 @@ vhost_user_if_input (vlib_main_t * vm,
 	    }
 	}
 
-      if (PREDICT_TRUE (vui->is_any_layout) ||
-	  (!(desc_table[desc_current].flags & VIRTQ_DESC_F_NEXT)))
-	{
-	  /* ANYLAYOUT or single buffer */
-	  desc_data_offset = vui->virtio_net_hdr_sz;
-	}
-      else
+      desc_data_offset = vui->virtio_net_hdr_sz;
+
+      if (enable_csum)
 	{
-	  /* CSR case without ANYLAYOUT, skip 1st buffer */
-	  desc_data_offset = desc_table[desc_current].len;
+	  virtio_net_hdr_mrg_rxbuf_t *hdr;
+	  u8 *b_data;
+	  u16 current;
+
+	  hdr = map_guest_mem (vui, desc_table[desc_current].addr, &map_hint);
+	  if (PREDICT_FALSE (hdr == 0))
+	    {
+	      vlib_error_count (vm, node->node_index,
+				VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL, 1);
+	      goto out;
+	    }
+	  if (hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
+	    {
+	      if ((desc_data_offset == desc_table[desc_current].len) &&
+		  (desc_table[desc_current].flags & VIRTQ_DESC_F_NEXT))
+		{
+		  current = desc_table[desc_current].next;
+		  b_data = map_guest_mem (vui, desc_table[current].addr,
+					  &map_hint);
+		  if (PREDICT_FALSE (b_data == 0))
+		    {
+		      vlib_error_count (vm, node->node_index,
+					VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL,
+					1);
+		      goto out;
+		    }
+		}
+	      else
+		b_data = (u8 *) hdr + desc_data_offset;
+
+	      vhost_user_handle_rx_offload (b_head, b_data, &hdr->hdr);
+	    }
 	}
 
       while (1)
@@ -497,8 +613,7 @@ vhost_user_if_input (vlib_main_t * vm,
 	    }
 
 	  /* Get more output if necessary. Or end of packet. */
-	  if (PREDICT_FALSE
-	      (b_current->current_length == VLIB_BUFFER_DATA_SIZE))
+	  if (PREDICT_FALSE (b_current->current_length == buffer_data_size))
 	    {
 	      if (PREDICT_FALSE (cpu->rx_buffers_len == 0))
 		{
@@ -528,10 +643,11 @@ vhost_user_if_input (vlib_main_t * vm,
 	    }
 
 	  /* Prepare a copy order executed later for the data */
+	  ASSERT (copy_len < VHOST_USER_COPY_ARRAY_N);
 	  vhost_copy_t *cpy = &cpu->copy[copy_len];
 	  copy_len++;
 	  u32 desc_data_l = desc_table[desc_current].len - desc_data_offset;
-	  cpy->len = VLIB_BUFFER_DATA_SIZE - b_current->current_length;
+	  cpy->len = buffer_data_size - b_current->current_length;
 	  cpy->len = (cpy->len > desc_data_l) ? desc_data_l : cpy->len;
 	  cpy->dst = (uword) (vlib_buffer_get_current (b_current) +
 			      b_current->current_length);
@@ -552,8 +668,8 @@ vhost_user_if_input (vlib_main_t * vm,
 	b_head->current_length;
 
       /* consume the descriptor and return it as used */
-      txvq->last_avail_idx++;
-      txvq->last_used_idx++;
+      last_avail_idx++;
+      last_used_idx++;
 
       VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b_head);
 
@@ -585,14 +701,17 @@ vhost_user_if_input (vlib_main_t * vm,
 	  copy_len = 0;
 
 	  /* give buffers back to driver */
-	  CLIB_MEMORY_BARRIER ();
-	  txvq->used->idx = txvq->last_used_idx;
+	  CLIB_MEMORY_STORE_BARRIER ();
+	  txvq->used->idx = last_used_idx;
 	  vhost_user_log_dirty_ring (vui, txvq, idx);
 	}
     }
 stop:
   vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 
+  txvq->last_used_idx = last_used_idx;
+  txvq->last_avail_idx = last_avail_idx;
+
   /* Do the memory copies */
   if (PREDICT_FALSE (vhost_user_input_copy (vui, cpu->copy, copy_len,
 					    &map_hint)))
@@ -602,7 +721,7 @@ stop:
     }
 
   /* give buffers back to driver */
-  CLIB_MEMORY_BARRIER ();
+  CLIB_MEMORY_STORE_BARRIER ();
   txvq->used->idx = txvq->last_used_idx;
   vhost_user_log_dirty_ring (vui, txvq, idx);
 
@@ -624,6 +743,7 @@ stop:
 
   vnet_device_increment_rx_packets (vm->thread_index, n_rx_packets);
 
+done:
   return n_rx_packets;
 }
 
@@ -645,8 +765,14 @@ VLIB_NODE_FN (vhost_user_input_node) (vlib_main_t * vm,
       {
 	vui =
 	  pool_elt_at_index (vum->vhost_user_interfaces, dq->dev_instance);
-	n_rx_packets += vhost_user_if_input (vm, vum, vui, dq->queue_id, node,
-					     dq->mode);
+	if (vui->features & (1ULL << FEAT_VIRTIO_NET_F_CSUM))
+	  n_rx_packets +=
+	    vhost_user_if_input (vm, vum, vui, dq->queue_id, node, dq->mode,
+				 1);
+	else
+	  n_rx_packets +=
+	    vhost_user_if_input (vm, vum, vui, dq->queue_id, node, dq->mode,
+				 0);
       }
   }
 
@@ -658,6 +784,7 @@ VLIB_REGISTER_NODE (vhost_user_input_node) = {
   .type = VLIB_NODE_TYPE_INPUT,
   .name = "vhost-user-input",
   .sibling_of = "device-input",
+  .flags = VLIB_NODE_FLAG_TRACE_SUPPORTED,
 
   /* Will be enabled if/when hardware is detected. */
   .state = VLIB_NODE_STATE_DISABLED,