2 *------------------------------------------------------------------
3 * Copyright (c) 2018 Cisco and/or its affiliates.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 *------------------------------------------------------------------
18 #include <vlib/vlib.h>
19 #include <vlib/unix/unix.h>
20 #include <vlib/pci/pci.h>
21 #include <vnet/ethernet/ethernet.h>
22 #include <vnet/devices/devices.h>
23 #include <vnet/ip/ip6_packet.h>
24 #include <vnet/ip/ip4_packet.h>
26 #include <vmxnet3/vmxnet3.h>
28 static_always_inline void
29 vmxnet3_tx_comp_ring_advance_next (vmxnet3_txq_t * txq)
31 vmxnet3_tx_comp_ring *comp_ring = &txq->tx_comp_ring;
34 if (PREDICT_FALSE (comp_ring->next == txq->size))
37 comp_ring->gen ^= VMXNET3_TXCF_GEN;
41 static_always_inline void
42 vmxnet3_tx_ring_advance_produce (vmxnet3_txq_t * txq)
44 txq->tx_ring.produce++;
45 if (PREDICT_FALSE (txq->tx_ring.produce == txq->size))
47 txq->tx_ring.produce = 0;
48 txq->tx_ring.gen ^= VMXNET3_TXF_GEN;
52 static_always_inline void
53 vmxnet3_tx_ring_advance_consume (vmxnet3_txq_t * txq)
55 txq->tx_ring.consume++;
56 txq->tx_ring.consume &= txq->size - 1;
59 static_always_inline void
60 vmxnet3_txq_release (vlib_main_t * vm, vmxnet3_device_t * vd,
63 vmxnet3_tx_comp *tx_comp;
64 vmxnet3_tx_comp_ring *comp_ring;
66 comp_ring = &txq->tx_comp_ring;
67 tx_comp = &txq->tx_comp[comp_ring->next];
69 while ((tx_comp->flags & VMXNET3_TXCF_GEN) == comp_ring->gen)
71 u16 eop_idx = tx_comp->index & VMXNET3_TXC_INDEX;
72 u32 bi0 = txq->tx_ring.bufs[txq->tx_ring.consume];
74 vlib_buffer_free_one (vm, bi0);
75 while (txq->tx_ring.consume != eop_idx)
77 vmxnet3_tx_ring_advance_consume (txq);
79 vmxnet3_tx_ring_advance_consume (txq);
81 vmxnet3_tx_comp_ring_advance_next (txq);
82 tx_comp = &txq->tx_comp[comp_ring->next];
86 static_always_inline u16
87 vmxnet3_tx_ring_space_left (vmxnet3_txq_t * txq)
91 count = (txq->tx_ring.consume - txq->tx_ring.produce - 1);
93 if (txq->tx_ring.produce >= txq->tx_ring.consume)
98 VNET_DEVICE_CLASS_TX_FN (vmxnet3_device_class) (vlib_main_t * vm,
99 vlib_node_runtime_t * node,
100 vlib_frame_t * frame)
102 vmxnet3_main_t *vmxm = &vmxnet3_main;
103 vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
104 vmxnet3_device_t *vd = pool_elt_at_index (vmxm->devices, rd->dev_instance);
105 u32 *buffers = vlib_frame_vector_args (frame);
108 vmxnet3_tx_desc *txd = 0;
109 u32 desc_idx, generation, first_idx;
111 u16 n_left = frame->n_vectors;
113 u16 qid = vm->thread_index % vd->num_tx_queues, produce;
115 if (PREDICT_FALSE (!(vd->flags & VMXNET3_DEVICE_F_LINK_UP)))
117 vlib_buffer_free (vm, buffers, n_left);
118 vlib_error_count (vm, node->node_index, VMXNET3_TX_ERROR_LINK_DOWN,
123 txq = vec_elt_at_index (vd->txqs, qid);
124 clib_spinlock_lock_if_init (&txq->lock);
126 vmxnet3_txq_release (vm, vd, txq);
128 produce = txq->tx_ring.produce;
129 while (PREDICT_TRUE (n_left))
131 u16 space_needed = 1, i;
138 b0 = vlib_get_buffer (vm, bi0);
141 space_left = vmxnet3_tx_ring_space_left (txq);
142 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
144 u32 next_buffer = b->next_buffer;
146 b = vlib_get_buffer (vm, next_buffer);
149 if (PREDICT_FALSE (space_left < space_needed))
151 vmxnet3_txq_release (vm, vd, txq);
152 space_left = vmxnet3_tx_ring_space_left (txq);
154 if (PREDICT_FALSE (space_left < space_needed))
156 vlib_buffer_free_one (vm, bi0);
157 vlib_error_count (vm, node->node_index,
158 VMXNET3_TX_ERROR_NO_FREE_SLOTS, 1);
162 * Drop this packet. But we may have enough room for the next
170 * Toggle the generation bit for SOP fragment to avoid device starts
171 * reading incomplete packet
173 generation = txq->tx_ring.gen ^ VMXNET3_TXF_GEN;
174 first_idx = txq->tx_ring.produce;
175 for (i = 0; i < space_needed; i++)
177 b0 = vlib_get_buffer (vm, bi0);
178 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
180 desc_idx = txq->tx_ring.produce;
182 vmxnet3_tx_ring_advance_produce (txq);
183 txq->tx_ring.bufs[desc_idx] = bi0;
185 txd = &txq->tx_desc[desc_idx];
187 txd->address = vlib_buffer_get_current_pa (vm, b0);
189 txd->flags[0] = generation | b0->current_length;
191 if (PREDICT_FALSE (b0->flags & VNET_BUFFER_F_GSO))
194 * We should not be getting GSO outbound traffic unless it is
197 ASSERT (vd->gso_enable == 1);
198 gso_size = vnet_buffer2 (b0)->gso_size;
199 l4_hdr_sz = vnet_buffer2 (b0)->gso_l4_hdr_sz;
200 if (b0->flags & VNET_BUFFER_F_IS_IP6)
201 hdr_len = sizeof (ethernet_header_t) + sizeof (ip6_header_t) +
204 hdr_len = sizeof (ethernet_header_t) + sizeof (ip4_header_t) +
208 generation = txq->tx_ring.gen;
209 bi0 = b0->next_buffer;
211 if (PREDICT_FALSE (gso_size != 0))
213 txq->tx_desc[first_idx].flags[1] = hdr_len;
214 txq->tx_desc[first_idx].flags[1] |= VMXNET3_TXF_OM (VMXNET3_OM_TSO);
215 txq->tx_desc[first_idx].flags[0] |= VMXNET3_TXF_MSSCOF (gso_size);
217 txd->flags[1] |= VMXNET3_TXF_CQ | VMXNET3_TXF_EOP;
218 asm volatile ("":::"memory");
220 * Now toggle back the generation bit for the first segment.
221 * Device can start reading the packet
223 txq->tx_desc[first_idx].flags[0] ^= VMXNET3_TXF_GEN;
229 if (PREDICT_TRUE (produce != txq->tx_ring.produce))
230 vmxnet3_reg_write_inline (vd, 0, txq->reg_txprod, txq->tx_ring.produce);
232 clib_spinlock_unlock_if_init (&txq->lock);
234 return (frame->n_vectors - n_left);
238 * fd.io coding-style-patch-verification: ON
241 * eval: (c-set-style "gnu")