virtio: add packet buffering on tx

[vpp.git] / src / vnet / devices / virtio / node.c

diff --git a/src/vnet/devices/virtio/node.c b/src/vnet/devices/virtio/node.c
index 9611271..42b2590 100644 (file)
--- a/src/vnet/devices/virtio/node.c
+++ b/src/vnet/devices/virtio/node.c
@@ -28,6 +28,7 @@
 #include <vnet/ethernet/ethernet.h>
 #include <vnet/devices/devices.h>
 #include <vnet/feature/feature.h>
+#include <vnet/gso/gro_func.h>
 #include <vnet/ip/ip4_packet.h>
 #include <vnet/ip/ip6_packet.h>
 #include <vnet/udp/udp_packet.h>
@@ -35,6 +36,7 @@
 
 
 #define foreach_virtio_input_error \
+  _(BUFFER_ALLOC, "buffer alloc error") \
   _(UNKNOWN, "unknown")
 
 typedef enum
@@ -57,7 +59,7 @@ typedef struct
   u32 hw_if_index;
   u16 ring;
   u16 len;
-  struct virtio_net_hdr_v1 hdr;
+  virtio_net_hdr_v1_t hdr;
 } virtio_input_trace_t;
 
 static u8 *
@@ -81,9 +83,9 @@ format_virtio_input_trace (u8 * s, va_list * args)
 static_always_inline void
 virtio_refill_vring (vlib_main_t * vm, virtio_if_t * vif,
                     virtio_if_type_t type, virtio_vring_t * vring,
-                    const int hdr_sz)
+                    const int hdr_sz, u32 node_index)
 {
-  u16 used, next, avail, n_slots;
+  u16 used, next, avail, n_slots, n_refill;
   u16 sz = vring->size;
   u16 mask = sz - 1;
 
@@ -94,21 +96,26 @@ more:
     return;
 
   /* deliver free buffers in chunks of 64 */
-  n_slots = clib_min (sz - used, 64);
+  n_refill = clib_min (sz - used, 64);
 
   next = vring->desc_next;
   avail = vring->avail->idx;
   n_slots =
     vlib_buffer_alloc_to_ring_from_pool (vm, vring->buffers, next,
-                                        vring->size, n_slots,
+                                        vring->size, n_refill,
                                         vring->buffer_pool_index);
 
-  if (n_slots == 0)
-    return;
+  if (PREDICT_FALSE (n_slots != n_refill))
+    {
+      vlib_error_count (vm, node_index,
+                       VIRTIO_INPUT_ERROR_BUFFER_ALLOC, n_refill - n_slots);
+      if (n_slots == 0)
+       return;
+    }
 
   while (n_slots)
     {
-      struct vring_desc *d = &vring->desc[next];;
+      vring_desc_t *d = &vring->desc[next];;
       vlib_buffer_t *b = vlib_get_buffer (vm, vring->buffers[next]);
       /*
        * current_data may not be initialized with 0 and may contain
@@ -134,7 +141,7 @@ more:
   vring->desc_next = next;
   vring->desc_in_use = used;
 
-  if ((vring->used->flags & VIRTIO_RING_FLAG_MASK_INT) == 0)
+  if ((vring->used->flags & VRING_USED_F_NO_NOTIFY) == 0)
     {
       virtio_kick (vm, vring, vif);
     }
@@ -142,7 +149,7 @@ more:
 }
 
 static_always_inline void
-virtio_needs_csum (vlib_buffer_t * b0, struct virtio_net_hdr_v1 *hdr,
+virtio_needs_csum (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr,
                   u8 * l4_proto, u8 * l4_hdr_sz, virtio_if_type_t type)
 {
   if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
@@ -232,7 +239,7 @@ virtio_needs_csum (vlib_buffer_t * b0, struct virtio_net_hdr_v1 *hdr,
 }
 
 static_always_inline void
-fill_gso_buffer_flags (vlib_buffer_t * b0, struct virtio_net_hdr_v1 *hdr,
+fill_gso_buffer_flags (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr,
                       u8 l4_proto, u8 l4_hdr_sz)
 {
   if (hdr->gso_type == VIRTIO_NET_HDR_GSO_TCPV4)
@@ -261,6 +268,8 @@ virtio_device_input_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
   u32 thread_index = vm->thread_index;
   uword n_trace = vlib_get_trace_count (vm, node);
   virtio_vring_t *vring = vec_elt_at_index (vif->rxq_vrings, qid);
+  u16 txq_id = thread_index % vif->num_txqs;
+  virtio_vring_t *txq_vring = vec_elt_at_index (vif->txq_vrings, txq_id);
   u32 next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
   const int hdr_sz = vif->virtio_net_hdr_sz;
   u32 *to_next = 0;
@@ -270,7 +279,18 @@ virtio_device_input_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
   u16 last = vring->last_used_idx;
   u16 n_left = vring->used->idx - last;
 
-  if ((vring->used->flags & VIRTIO_RING_FLAG_MASK_INT) == 0 &&
+  if (clib_spinlock_trylock_if_init (&txq_vring->lockp))
+    {
+      if (vif->packet_coalesce)
+       vnet_gro_flow_table_schedule_node_on_dispatcher (vm,
+                                                        txq_vring->flow_table);
+      else if (vif->packet_buffering)
+       virtio_vring_buffering_schedule_node_on_dispatcher (vm,
+                                                           txq_vring->buffering);
+      clib_spinlock_unlock_if_init (&txq_vring->lockp);
+    }
+
+  if ((vring->used->flags & VRING_USED_F_NO_NOTIFY) == 0 &&
       vring->last_kick_avail_idx != vring->avail->idx)
     virtio_kick (vm, vring, vif);
 
@@ -291,14 +311,14 @@ virtio_device_input_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
        {
          u8 l4_proto = 0, l4_hdr_sz = 0;
          u16 num_buffers = 1;
-         struct vring_used_elem *e = &vring->used->ring[last & mask];
-         struct virtio_net_hdr_v1 *hdr;
+         vring_used_elem_t *e = &vring->used->ring[last & mask];
+         virtio_net_hdr_v1_t *hdr;
          u16 slot = e->id;
          u16 len = e->len - hdr_sz;
          u32 bi0 = vring->buffers[slot];
          vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0);
          hdr = vlib_buffer_get_current (b0);
-         if (hdr_sz == sizeof (struct virtio_net_hdr_v1))
+         if (hdr_sz == sizeof (virtio_net_hdr_v1_t))
            num_buffers = hdr->num_buffers;
 
          b0->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID;
@@ -413,7 +433,7 @@ virtio_device_input_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
                                   n_rx_bytes);
 
 refill:
-  virtio_refill_vring (vm, vif, type, vring, hdr_sz);
+  virtio_refill_vring (vm, vif, type, vring, hdr_sz, node->node_index);
 
   return n_rx_packets;
 }