2 *------------------------------------------------------------------
3 * Copyright (c) 2016 Cisco and/or its affiliates.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 *------------------------------------------------------------------
18 #include <sys/types.h>
22 #include <vlib/vlib.h>
23 #include <vlib/unix/unix.h>
24 #include <vnet/ethernet/ethernet.h>
25 #include <vnet/ip/ip4_packet.h>
26 #include <vnet/ip/ip6_packet.h>
27 #include <vnet/devices/virtio/virtio.h>
29 #define foreach_virtio_tx_func_error \
30 _(NO_FREE_SLOTS, "no free tx slots") \
31 _(TRUNC_PACKET, "packet > buffer size -- truncated in tx ring") \
32 _(PENDING_MSGS, "pending msgs in tx ring") \
33 _(NO_TX_QUEUES, "no tx queues")
37 #define _(f,s) VIRTIO_TX_ERROR_##f,
38 foreach_virtio_tx_func_error
41 } virtio_tx_func_error_t;
43 static char *virtio_tx_func_error_strings[] = {
45 foreach_virtio_tx_func_error
49 #ifndef CLIB_MARCH_VARIANT
51 format_virtio_device_name (u8 * s, va_list * args)
53 u32 dev_instance = va_arg (*args, u32);
54 virtio_main_t *mm = &virtio_main;
55 virtio_if_t *vif = pool_elt_at_index (mm->interfaces, dev_instance);
57 if (vif->type == VIRTIO_IF_TYPE_TAP)
58 s = format (s, "tap%u", vif->id);
59 else if (vif->type == VIRTIO_IF_TYPE_PCI)
60 s = format (s, "virtio-%x/%x/%x/%x", vif->pci_addr.domain,
61 vif->pci_addr.bus, vif->pci_addr.slot,
62 vif->pci_addr.function);
64 s = format (s, "virtio-%lu", vif->dev_instance);
68 #endif /* CLIB_MARCH_VARIANT */
71 format_virtio_device (u8 * s, va_list * args)
73 u32 dev_instance = va_arg (*args, u32);
74 int verbose = va_arg (*args, int);
75 u32 indent = format_get_indent (s);
77 s = format (s, "VIRTIO interface");
80 s = format (s, "\n%U instance %u", format_white_space, indent + 2,
87 format_virtio_tx_trace (u8 * s, va_list * args)
89 s = format (s, "Unimplemented...");
93 static_always_inline void
94 virtio_free_used_device_desc (vlib_main_t * vm, virtio_vring_t * vring)
96 u16 used = vring->desc_in_use;
99 u16 last = vring->last_used_idx;
100 u16 n_left = vring->used->idx - last;
107 struct vring_used_elem *e = &vring->used->ring[last & mask];
109 slot = n_buffers = e->id;
111 while (e->id == n_buffers)
118 e = &vring->used->ring[last & mask];
120 vlib_buffer_free_from_ring (vm, vring->buffers, slot,
121 sz, (n_buffers - slot));
122 used -= (n_buffers - slot);
128 vlib_buffer_free (vm, &vring->buffers[slot], 1);
134 vring->desc_in_use = used;
135 vring->last_used_idx = last;
138 static_always_inline u16
139 add_buffer_to_slot (vlib_main_t * vm, virtio_if_t * vif,
140 virtio_vring_t * vring, u32 bi, u16 avail, u16 next,
141 u16 mask, int do_gso)
144 int hdr_sz = vif->virtio_net_hdr_sz;
145 struct vring_desc *d;
146 d = &vring->desc[next];
147 vlib_buffer_t *b = vlib_get_buffer (vm, bi);
148 struct virtio_net_hdr_v1 *hdr = vlib_buffer_get_current (b) - hdr_sz;
150 clib_memset (hdr, 0, hdr_sz);
151 if (do_gso && (b->flags & VNET_BUFFER_F_GSO))
153 if (b->flags & VNET_BUFFER_F_IS_IP4)
155 hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
156 hdr->gso_size = vnet_buffer2 (b)->gso_size;
157 hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
158 hdr->csum_start = vnet_buffer (b)->l4_hdr_offset; // 0x22;
159 hdr->csum_offset = 0x10;
163 hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
164 hdr->gso_size = vnet_buffer2 (b)->gso_size;
165 hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
166 hdr->csum_start = vnet_buffer (b)->l4_hdr_offset; // 0x36;
167 hdr->csum_offset = 0x10;
171 if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
174 ((vif->type == VIRTIO_IF_TYPE_PCI) ? vlib_buffer_get_current_pa (vm,
176 pointer_to_uword (vlib_buffer_get_current (b))) - hdr_sz;
177 d->len = b->current_length + hdr_sz;
183 * We are using single vlib_buffer_t for indirect descriptor(s)
184 * chain. Single descriptor is 16 bytes and vlib_buffer_t
185 * has 2048 bytes space. So maximum long chain can have 128
186 * (=2048/16) indirect descriptors.
187 * It can easily support 65535 bytes of Jumbo frames with
188 * each data buffer size of 512 bytes minimum.
190 u32 indirect_buffer = 0;
191 if (PREDICT_FALSE (vlib_buffer_alloc (vm, &indirect_buffer, 1) == 0))
194 vlib_buffer_t *indirect_desc = vlib_get_buffer (vm, indirect_buffer);
195 indirect_desc->current_data = 0;
196 indirect_desc->flags |= VLIB_BUFFER_NEXT_PRESENT;
197 indirect_desc->next_buffer = bi;
198 bi = indirect_buffer;
200 struct vring_desc *id =
201 (struct vring_desc *) vlib_buffer_get_current (indirect_desc);
203 if (vif->type == VIRTIO_IF_TYPE_PCI)
205 d->addr = vlib_physmem_get_pa (vm, id);
206 id->addr = vlib_buffer_get_current_pa (vm, b) - hdr_sz;
209 * If VIRTIO_F_ANY_LAYOUT is not negotiated, then virtio_net_hdr
210 * should be presented in separate descriptor and data will start
211 * from next descriptor.
214 (vif->features & VIRTIO_FEATURE (VIRTIO_F_ANY_LAYOUT)))
215 id->len = b->current_length + hdr_sz;
219 id->flags = VRING_DESC_F_NEXT;
223 id->addr = vlib_buffer_get_current_pa (vm, b);
224 id->len = b->current_length;
226 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
228 id->flags = VRING_DESC_F_NEXT;
232 b = vlib_get_buffer (vm, b->next_buffer);
233 id->addr = vlib_buffer_get_current_pa (vm, b);
234 id->len = b->current_length;
237 else /* VIRTIO_IF_TYPE_TAP */
239 d->addr = pointer_to_uword (id);
240 /* first buffer in chain */
241 id->addr = pointer_to_uword (vlib_buffer_get_current (b)) - hdr_sz;
242 id->len = b->current_length + hdr_sz;
244 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
246 id->flags = VRING_DESC_F_NEXT;
250 b = vlib_get_buffer (vm, b->next_buffer);
251 id->addr = pointer_to_uword (vlib_buffer_get_current (b));
252 id->len = b->current_length;
257 d->len = count * sizeof (struct vring_desc);
258 d->flags = VRING_DESC_F_INDIRECT;
260 vring->buffers[next] = bi;
261 vring->avail->ring[avail & mask] = next;
266 static_always_inline uword
267 virtio_interface_tx_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
268 vlib_frame_t * frame, virtio_if_t * vif,
271 u16 n_left = frame->n_vectors;
272 virtio_vring_t *vring;
273 u16 qid = vm->thread_index % vif->num_txqs;
274 vring = vec_elt_at_index (vif->txq_vrings, qid);
275 u16 used, next, avail;
276 u16 sz = vring->size;
278 u32 *buffers = vlib_frame_vector_args (frame);
280 clib_spinlock_lock_if_init (&vring->lockp);
282 if ((vring->used->flags & VIRTIO_RING_FLAG_MASK_INT) == 0 &&
283 (vring->last_kick_avail_idx != vring->avail->idx))
284 virtio_kick (vm, vring, vif);
286 /* free consumed buffers */
287 virtio_free_used_device_desc (vm, vring);
289 used = vring->desc_in_use;
290 next = vring->desc_next;
291 avail = vring->avail->idx;
293 while (n_left && used < sz)
297 add_buffer_to_slot (vm, vif, vring, buffers[0], avail, next, mask,
302 next = (next + n_added) & mask;
308 if (n_left != frame->n_vectors)
310 CLIB_MEMORY_STORE_BARRIER ();
311 vring->avail->idx = avail;
312 vring->desc_next = next;
313 vring->desc_in_use = used;
314 if ((vring->used->flags & VIRTIO_RING_FLAG_MASK_INT) == 0)
315 virtio_kick (vm, vring, vif);
320 vlib_error_count (vm, node->node_index, VIRTIO_TX_ERROR_NO_FREE_SLOTS,
322 vlib_buffer_free (vm, buffers, n_left);
325 clib_spinlock_unlock_if_init (&vring->lockp);
327 return frame->n_vectors - n_left;
330 VNET_DEVICE_CLASS_TX_FN (virtio_device_class) (vlib_main_t * vm,
331 vlib_node_runtime_t * node,
332 vlib_frame_t * frame)
334 virtio_main_t *nm = &virtio_main;
335 vnet_interface_output_runtime_t *rund = (void *) node->runtime_data;
336 virtio_if_t *vif = pool_elt_at_index (nm->interfaces, rund->dev_instance);
337 vnet_main_t *vnm = vnet_get_main ();
339 if (vnm->interface_main.gso_interface_count > 0)
340 return virtio_interface_tx_inline (vm, node, frame, vif, 1 /* do_gso */ );
342 return virtio_interface_tx_inline (vm, node, frame, vif,
347 virtio_set_interface_next_node (vnet_main_t * vnm, u32 hw_if_index,
350 virtio_main_t *apm = &virtio_main;
351 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
352 virtio_if_t *vif = pool_elt_at_index (apm->interfaces, hw->dev_instance);
354 /* Shut off redirection */
355 if (node_index == ~0)
357 vif->per_interface_next_index = node_index;
361 vif->per_interface_next_index =
362 vlib_node_add_next (vlib_get_main (), virtio_input_node.index,
367 virtio_clear_hw_interface_counters (u32 instance)
369 /* Nothing for now */
372 static clib_error_t *
373 virtio_interface_rx_mode_change (vnet_main_t * vnm, u32 hw_if_index, u32 qid,
374 vnet_hw_interface_rx_mode mode)
376 virtio_main_t *mm = &virtio_main;
377 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
378 virtio_if_t *vif = pool_elt_at_index (mm->interfaces, hw->dev_instance);
379 virtio_vring_t *vring = vec_elt_at_index (vif->rxq_vrings, qid);
381 if (vif->type == VIRTIO_IF_TYPE_PCI && !(vif->support_int_mode))
383 vring->avail->flags |= VIRTIO_RING_FLAG_MASK_INT;
384 return clib_error_return (0, "interrupt mode is not supported");
387 if (mode == VNET_HW_INTERFACE_RX_MODE_POLLING)
388 vring->avail->flags |= VIRTIO_RING_FLAG_MASK_INT;
390 vring->avail->flags &= ~VIRTIO_RING_FLAG_MASK_INT;
395 static clib_error_t *
396 virtio_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags)
398 virtio_main_t *mm = &virtio_main;
399 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
400 virtio_if_t *vif = pool_elt_at_index (mm->interfaces, hw->dev_instance);
402 if (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP)
403 vif->flags |= VIRTIO_IF_FLAG_ADMIN_UP;
405 vif->flags &= ~VIRTIO_IF_FLAG_ADMIN_UP;
410 static clib_error_t *
411 virtio_subif_add_del_function (vnet_main_t * vnm,
413 struct vnet_sw_interface_t *st, int is_add)
415 /* Nothing for now */
420 VNET_DEVICE_CLASS (virtio_device_class) = {
422 .format_device_name = format_virtio_device_name,
423 .format_device = format_virtio_device,
424 .format_tx_trace = format_virtio_tx_trace,
425 .tx_function_n_errors = VIRTIO_TX_N_ERROR,
426 .tx_function_error_strings = virtio_tx_func_error_strings,
427 .rx_redirect_to_node = virtio_set_interface_next_node,
428 .clear_counters = virtio_clear_hw_interface_counters,
429 .admin_up_down_function = virtio_interface_admin_up_down,
430 .subif_add_del_function = virtio_subif_add_del_function,
431 .rx_mode_change_function = virtio_interface_rx_mode_change,
436 * fd.io coding-style-patch-verification: ON
439 * eval: (c-set-style "gnu")