vlib: fix trace number accounting

[vpp.git] / src / vnet / devices / virtio / node.c

diff --git a/src/vnet/devices/virtio/node.c b/src/vnet/devices/virtio/node.c
index 1c9cfd0..91e788e 100644 (file)
--- a/src/vnet/devices/virtio/node.c
+++ b/src/vnet/devices/virtio/node.c
@@ -261,16 +261,13 @@ fill_gso_buffer_flags (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr,
 static_always_inline uword
 virtio_device_input_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
                                vlib_frame_t * frame, virtio_if_t * vif,
-                               u16 qid, virtio_if_type_t type,
+                               virtio_vring_t * vring, virtio_if_type_t type,
                                int gso_enabled, int checksum_offload_enabled)
 {
   vnet_main_t *vnm = vnet_get_main ();
   u32 thread_index = vm->thread_index;
   uword n_trace = vlib_get_trace_count (vm, node);
-  virtio_vring_t *vring = vec_elt_at_index (vif->rxq_vrings, qid);
-  u16 txq_id = thread_index % vif->num_txqs;
-  virtio_vring_t *txq_vring = vec_elt_at_index (vif->txq_vrings, txq_id);
-  u32 next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
+  u32 next_index;
   const int hdr_sz = vif->virtio_net_hdr_sz;
   u32 *to_next = 0;
   u32 n_rx_packets = 0;
@@ -278,24 +275,24 @@ virtio_device_input_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
   u16 mask = vring->size - 1;
   u16 last = vring->last_used_idx;
   u16 n_left = vring->used->idx - last;
-
-  if (vif->packet_coalesce
-      && clib_spinlock_trylock_if_init (&txq_vring->lockp))
-    {
-      vnet_gro_flow_table_schedule_node_on_dispatcher (vm,
-                                                      txq_vring->flow_table);
-      clib_spinlock_unlock_if_init (&txq_vring->lockp);
-    }
-
-  if ((vring->used->flags & VRING_USED_F_NO_NOTIFY) == 0 &&
-      vring->last_kick_avail_idx != vring->avail->idx)
-    virtio_kick (vm, vring, vif);
+  vlib_buffer_t bt;
 
   if (n_left == 0)
-    goto refill;
+    return 0;
 
   if (type == VIRTIO_IF_TYPE_TUN)
-    next_index = VNET_DEVICE_INPUT_NEXT_IP4_INPUT;
+    {
+      next_index = VNET_DEVICE_INPUT_NEXT_IP4_INPUT;
+    }
+  else
+    {
+      next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
+      if (PREDICT_FALSE (vif->per_interface_next_index != ~0))
+       next_index = vif->per_interface_next_index;
+
+      /* only for l2, redirect if feature path enabled */
+      vnet_feature_start_device_input_x1 (vif->sw_if_index, &next_index, &bt);
+    }
 
   while (n_left)
     {
@@ -376,26 +373,25 @@ virtio_device_input_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
                  next0 = VNET_DEVICE_INPUT_NEXT_DROP;
                  break;
                }
-           }
 
-         if (PREDICT_FALSE (vif->per_interface_next_index != ~0))
-           next0 = vif->per_interface_next_index;
-
-         if (type != VIRTIO_IF_TYPE_TUN)
+             if (PREDICT_FALSE (vif->per_interface_next_index != ~0))
+               next0 = vif->per_interface_next_index;
+           }
+         else
            {
-             /* only for l2, redirect if feature path enabled */
-             vnet_feature_start_device_input_x1 (vif->sw_if_index, &next0,
-                                                 b0);
+             /* copy feature arc data from template */
+             b0->current_config_index = bt.current_config_index;
+             vnet_buffer (b0)->feature_arc_index =
+               vnet_buffer (&bt)->feature_arc_index;
            }
 
          /* trace */
          VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
 
-         if (PREDICT_FALSE (n_trace > 0))
+         if (PREDICT_FALSE (n_trace > 0 && vlib_trace_buffer (vm, node, next0, b0,     /* follow_chain */
+                                                              1)))
            {
              virtio_input_trace_t *tr;
-             vlib_trace_buffer (vm, node, next0, b0,
-                                /* follow_chain */ 1);
              vlib_set_trace_count (vm, node, --n_trace);
              tr = vlib_add_trace (vm, node, b0, sizeof (*tr));
              tr->next_index = next0;
@@ -412,9 +408,10 @@ virtio_device_input_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
          n_left--;
          last++;
 
-         /* enqueue */
-         vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
-                                          n_left_to_next, bi0, next0);
+         /* only tun interfaces may have different next index */
+         if (type == VIRTIO_IF_TYPE_TUN)
+           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
+                                            n_left_to_next, bi0, next0);
 
          /* next packet */
          n_rx_packets++;
@@ -429,9 +426,6 @@ virtio_device_input_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
                                   vif->sw_if_index, n_rx_packets,
                                   n_rx_bytes);
 
-refill:
-  virtio_refill_vring (vm, vif, type, vring, hdr_sz, node->node_index);
-
   return n_rx_packets;
 }
 
@@ -440,17 +434,39 @@ virtio_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
                            vlib_frame_t * frame, virtio_if_t * vif, u16 qid,
                            virtio_if_type_t type)
 {
+  virtio_vring_t *vring = vec_elt_at_index (vif->rxq_vrings, qid);
+  const int hdr_sz = vif->virtio_net_hdr_sz;
+  u16 txq_id = vm->thread_index % vif->num_txqs;
+  virtio_vring_t *txq_vring = vec_elt_at_index (vif->txq_vrings, txq_id);
+  uword rv;
+
+  if (clib_spinlock_trylock_if_init (&txq_vring->lockp))
+    {
+      if (vif->packet_coalesce)
+       vnet_gro_flow_table_schedule_node_on_dispatcher
+         (vm, txq_vring->flow_table);
+      else if (vif->packet_buffering)
+       virtio_vring_buffering_schedule_node_on_dispatcher
+         (vm, txq_vring->buffering);
+      clib_spinlock_unlock_if_init (&txq_vring->lockp);
+    }
+
+  if ((vring->used->flags & VRING_USED_F_NO_NOTIFY) == 0 &&
+      vring->last_kick_avail_idx != vring->avail->idx)
+    virtio_kick (vm, vring, vif);
 
   if (vif->gso_enabled)
-    return virtio_device_input_gso_inline (vm, node, frame, vif,
-                                          qid, type, 1, 1);
+    rv = virtio_device_input_gso_inline (vm, node, frame, vif, vring, type,
+                                        1, 1);
   else if (vif->csum_offload_enabled)
-    return virtio_device_input_gso_inline (vm, node, frame, vif,
-                                          qid, type, 0, 1);
+    rv = virtio_device_input_gso_inline (vm, node, frame, vif, vring, type,
+                                        0, 1);
   else
-    return virtio_device_input_gso_inline (vm, node, frame, vif,
-                                          qid, type, 0, 0);
-  return 0;
+    rv = virtio_device_input_gso_inline (vm, node, frame, vif, vring, type,
+                                        0, 0);
+
+  virtio_refill_vring (vm, vif, type, vring, hdr_sz, node->node_index);
+  return rv;
 }
 
 VLIB_NODE_FN (virtio_input_node) (vlib_main_t * vm,