2 *------------------------------------------------------------------
3 * Copyright (c) 2016 Cisco and/or its affiliates.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 *------------------------------------------------------------------
18 #include <sys/types.h>
22 #include <vlib/vlib.h>
23 #include <vlib/unix/unix.h>
24 #include <vnet/ethernet/ethernet.h>
25 #include <vnet/ip/ip4_packet.h>
26 #include <vnet/ip/ip6_packet.h>
27 #include <vnet/devices/virtio/virtio.h>
29 #define foreach_virtio_tx_func_error \
30 _(NO_FREE_SLOTS, "no free tx slots") \
31 _(TRUNC_PACKET, "packet > buffer size -- truncated in tx ring") \
32 _(PENDING_MSGS, "pending msgs in tx ring") \
33 _(NO_TX_QUEUES, "no tx queues")
37 #define _(f,s) VIRTIO_TX_ERROR_##f,
38 foreach_virtio_tx_func_error
41 } virtio_tx_func_error_t;
43 static char *virtio_tx_func_error_strings[] = {
45 foreach_virtio_tx_func_error
49 #ifndef CLIB_MARCH_VARIANT
51 format_virtio_device_name (u8 * s, va_list * args)
53 u32 dev_instance = va_arg (*args, u32);
54 virtio_main_t *mm = &virtio_main;
55 virtio_if_t *vif = pool_elt_at_index (mm->interfaces, dev_instance);
57 if (vif->type == VIRTIO_IF_TYPE_TAP)
58 s = format (s, "tap%u", vif->id);
59 else if (vif->type == VIRTIO_IF_TYPE_PCI)
60 s = format (s, "virtio-%x/%x/%x/%x", vif->pci_addr.domain,
61 vif->pci_addr.bus, vif->pci_addr.slot,
62 vif->pci_addr.function);
64 s = format (s, "virtio-%lu", vif->dev_instance);
68 #endif /* CLIB_MARCH_VARIANT */
71 format_virtio_device (u8 * s, va_list * args)
73 u32 dev_instance = va_arg (*args, u32);
74 int verbose = va_arg (*args, int);
75 u32 indent = format_get_indent (s);
77 s = format (s, "VIRTIO interface");
80 s = format (s, "\n%U instance %u", format_white_space, indent + 2,
87 format_virtio_tx_trace (u8 * s, va_list * args)
89 s = format (s, "Unimplemented...");
93 #ifndef CLIB_MARCH_VARIANT
95 virtio_free_used_desc (vlib_main_t * vm, virtio_vring_t * vring)
97 u16 used = vring->desc_in_use;
100 u16 last = vring->last_used_idx;
101 u16 n_left = vring->used->idx - last;
108 struct vring_used_elem *e = &vring->used->ring[last & mask];
111 vlib_buffer_free (vm, &vring->buffers[slot], 1);
116 vring->desc_in_use = used;
117 vring->last_used_idx = last;
119 #endif /* CLIB_MARCH_VARIANT */
121 static_always_inline u16
122 add_buffer_to_slot (vlib_main_t * vm, virtio_if_t * vif,
123 virtio_vring_t * vring, u32 bi, u16 avail, u16 next,
124 u16 mask, int do_gso)
127 int hdr_sz = vif->virtio_net_hdr_sz;
128 struct vring_desc *d;
129 d = &vring->desc[next];
130 vlib_buffer_t *b = vlib_get_buffer (vm, bi);
131 struct virtio_net_hdr_v1 *hdr = vlib_buffer_get_current (b) - hdr_sz;
133 clib_memset (hdr, 0, hdr_sz);
134 if (do_gso && (b->flags & VNET_BUFFER_F_GSO))
136 if (b->flags & VNET_BUFFER_F_IS_IP4)
138 hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
139 hdr->gso_size = vnet_buffer2 (b)->gso_size;
140 hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
141 hdr->csum_start = vnet_buffer (b)->l4_hdr_offset; // 0x22;
142 hdr->csum_offset = 0x10;
146 hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
147 hdr->gso_size = vnet_buffer2 (b)->gso_size;
148 hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
149 hdr->csum_start = vnet_buffer (b)->l4_hdr_offset; // 0x36;
150 hdr->csum_offset = 0x10;
154 if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
157 ((vif->type == VIRTIO_IF_TYPE_PCI) ? vlib_buffer_get_current_pa (vm,
159 pointer_to_uword (vlib_buffer_get_current (b))) - hdr_sz;
160 d->len = b->current_length + hdr_sz;
166 * We are using single vlib_buffer_t for indirect descriptor(s)
167 * chain. Single descriptor is 16 bytes and vlib_buffer_t
168 * has 2048 bytes space. So maximum long chain can have 128
169 * (=2048/16) indirect descriptors.
170 * It can easily support 65535 bytes of Jumbo frames with
171 * each data buffer size of 512 bytes minimum.
173 vlib_buffer_t *indirect_desc =
174 vlib_get_buffer (vm, vring->indirect_buffers[next]);
175 indirect_desc->current_data = 0;
177 struct vring_desc *id =
178 (struct vring_desc *) vlib_buffer_get_current (indirect_desc);
180 if (vif->type == VIRTIO_IF_TYPE_PCI)
182 d->addr = vlib_physmem_get_pa (vm, id);
183 id->addr = vlib_buffer_get_current_pa (vm, b) - hdr_sz;
186 * If VIRTIO_F_ANY_LAYOUT is not negotiated, then virtio_net_hdr
187 * should be presented in separate descriptor and data will start
188 * from next descriptor.
191 (vif->features & VIRTIO_FEATURE (VIRTIO_F_ANY_LAYOUT)))
192 id->len = b->current_length + hdr_sz;
196 id->flags = VRING_DESC_F_NEXT;
200 id->addr = vlib_buffer_get_current_pa (vm, b);
201 id->len = b->current_length;
203 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
205 id->flags = VRING_DESC_F_NEXT;
209 b = vlib_get_buffer (vm, b->next_buffer);
210 id->addr = vlib_buffer_get_current_pa (vm, b);
211 id->len = b->current_length;
214 else /* VIRTIO_IF_TYPE_TAP */
216 d->addr = pointer_to_uword (id);
217 /* first buffer in chain */
218 id->addr = pointer_to_uword (vlib_buffer_get_current (b)) - hdr_sz;
219 id->len = b->current_length + hdr_sz;
221 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
223 id->flags = VRING_DESC_F_NEXT;
227 b = vlib_get_buffer (vm, b->next_buffer);
228 id->addr = pointer_to_uword (vlib_buffer_get_current (b));
229 id->len = b->current_length;
234 d->len = count * sizeof (struct vring_desc);
235 d->flags = VRING_DESC_F_INDIRECT;
237 vring->buffers[next] = bi;
238 vring->avail->ring[avail & mask] = next;
243 static_always_inline uword
244 virtio_interface_tx_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
245 vlib_frame_t * frame, virtio_if_t * vif,
248 u16 n_left = frame->n_vectors;
249 virtio_vring_t *vring;
250 u16 qid = vm->thread_index % vif->num_txqs;
251 vring = vec_elt_at_index (vif->txq_vrings, qid);
252 u16 used, next, avail;
253 u16 sz = vring->size;
255 u32 *buffers = vlib_frame_vector_args (frame);
257 clib_spinlock_lock_if_init (&vring->lockp);
259 if ((vring->used->flags & VIRTIO_RING_FLAG_MASK_INT) == 0 &&
260 (vring->last_kick_avail_idx != vring->avail->idx))
261 virtio_kick (vm, vring, vif);
263 /* free consumed buffers */
264 virtio_free_used_desc (vm, vring);
266 used = vring->desc_in_use;
267 next = vring->desc_next;
268 avail = vring->avail->idx;
270 while (n_left && used < sz)
274 add_buffer_to_slot (vm, vif, vring, buffers[0], avail, next, mask,
279 next = (next + n_added) & mask;
285 if (n_left != frame->n_vectors)
287 CLIB_MEMORY_STORE_BARRIER ();
288 vring->avail->idx = avail;
289 vring->desc_next = next;
290 vring->desc_in_use = used;
291 if ((vring->used->flags & VIRTIO_RING_FLAG_MASK_INT) == 0)
292 virtio_kick (vm, vring, vif);
297 vlib_error_count (vm, node->node_index, VIRTIO_TX_ERROR_NO_FREE_SLOTS,
299 vlib_buffer_free (vm, buffers, n_left);
302 clib_spinlock_unlock_if_init (&vring->lockp);
304 return frame->n_vectors - n_left;
307 VNET_DEVICE_CLASS_TX_FN (virtio_device_class) (vlib_main_t * vm,
308 vlib_node_runtime_t * node,
309 vlib_frame_t * frame)
311 virtio_main_t *nm = &virtio_main;
312 vnet_interface_output_runtime_t *rund = (void *) node->runtime_data;
313 virtio_if_t *vif = pool_elt_at_index (nm->interfaces, rund->dev_instance);
314 vnet_main_t *vnm = vnet_get_main ();
316 if (vnm->interface_main.gso_interface_count > 0)
317 return virtio_interface_tx_inline (vm, node, frame, vif, 1 /* do_gso */ );
319 return virtio_interface_tx_inline (vm, node, frame, vif,
324 virtio_set_interface_next_node (vnet_main_t * vnm, u32 hw_if_index,
327 virtio_main_t *apm = &virtio_main;
328 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
329 virtio_if_t *vif = pool_elt_at_index (apm->interfaces, hw->dev_instance);
331 /* Shut off redirection */
332 if (node_index == ~0)
334 vif->per_interface_next_index = node_index;
338 vif->per_interface_next_index =
339 vlib_node_add_next (vlib_get_main (), virtio_input_node.index,
344 virtio_clear_hw_interface_counters (u32 instance)
346 /* Nothing for now */
349 static clib_error_t *
350 virtio_interface_rx_mode_change (vnet_main_t * vnm, u32 hw_if_index, u32 qid,
351 vnet_hw_interface_rx_mode mode)
353 virtio_main_t *mm = &virtio_main;
354 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
355 virtio_if_t *vif = pool_elt_at_index (mm->interfaces, hw->dev_instance);
356 virtio_vring_t *vring = vec_elt_at_index (vif->rxq_vrings, qid);
358 if (vif->type == VIRTIO_IF_TYPE_PCI && !(vif->support_int_mode))
360 vring->avail->flags |= VIRTIO_RING_FLAG_MASK_INT;
361 return clib_error_return (0, "interrupt mode is not supported");
364 if (mode == VNET_HW_INTERFACE_RX_MODE_POLLING)
365 vring->avail->flags |= VIRTIO_RING_FLAG_MASK_INT;
367 vring->avail->flags &= ~VIRTIO_RING_FLAG_MASK_INT;
372 static clib_error_t *
373 virtio_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags)
375 virtio_main_t *mm = &virtio_main;
376 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
377 virtio_if_t *vif = pool_elt_at_index (mm->interfaces, hw->dev_instance);
379 if (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP)
380 vif->flags |= VIRTIO_IF_FLAG_ADMIN_UP;
382 vif->flags &= ~VIRTIO_IF_FLAG_ADMIN_UP;
387 static clib_error_t *
388 virtio_subif_add_del_function (vnet_main_t * vnm,
390 struct vnet_sw_interface_t *st, int is_add)
392 /* Nothing for now */
397 VNET_DEVICE_CLASS (virtio_device_class) = {
399 .format_device_name = format_virtio_device_name,
400 .format_device = format_virtio_device,
401 .format_tx_trace = format_virtio_tx_trace,
402 .tx_function_n_errors = VIRTIO_TX_N_ERROR,
403 .tx_function_error_strings = virtio_tx_func_error_strings,
404 .rx_redirect_to_node = virtio_set_interface_next_node,
405 .clear_counters = virtio_clear_hw_interface_counters,
406 .admin_up_down_function = virtio_interface_admin_up_down,
407 .subif_add_del_function = virtio_subif_add_del_function,
408 .rx_mode_change_function = virtio_interface_rx_mode_change,
413 * fd.io coding-style-patch-verification: ON
416 * eval: (c-set-style "gnu")