X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvnet%2Fdevices%2Fvirtio%2Fvirtio.c;h=99f581a1f1b318ffb5a0ab4f06bd0ca3046bac08;hb=b977d3f7c;hp=9fbbc1bf508f2c51dbe5947e9a0568f6d0e75530;hpb=286aa593a58df59e3a7d6f71bb2159dd439d7f76;p=vpp.git diff --git a/src/vnet/devices/virtio/virtio.c b/src/vnet/devices/virtio/virtio.c index 9fbbc1bf508..99f581a1f1b 100644 --- a/src/vnet/devices/virtio/virtio.c +++ b/src/vnet/devices/virtio/virtio.c @@ -21,8 +21,6 @@ #include #include #include -#include -#include #include #include @@ -55,7 +53,8 @@ call_read_ready (clib_file_t * uf) CLIB_UNUSED (ssize_t size) = read (uf->file_descriptor, &b, sizeof (b)); if ((qid & 1) == 0) - vnet_device_input_set_interrupt_pending (vnm, vif->hw_if_index, qid); + vnet_device_input_set_interrupt_pending (vnm, vif->hw_if_index, + RX_QUEUE_ACCESS (qid)); return 0; } @@ -64,11 +63,7 @@ call_read_ready (clib_file_t * uf) clib_error_t * virtio_vring_init (vlib_main_t * vm, virtio_if_t * vif, u16 idx, u16 sz) { - clib_error_t *err = 0; virtio_vring_t *vring; - struct vhost_vring_state state = { 0 }; - struct vhost_vring_addr addr = { 0 }; - struct vhost_vring_file file = { 0 }; clib_file_t t = { 0 }; int i; @@ -83,9 +78,12 @@ virtio_vring_init (vlib_main_t * vm, virtio_if_t * vif, u16 idx, u16 sz) if (idx % 2) { + vlib_thread_main_t *thm = vlib_get_thread_main (); vec_validate_aligned (vif->txq_vrings, TX_QUEUE_ACCESS (idx), CLIB_CACHE_LINE_BYTES); vring = vec_elt_at_index (vif->txq_vrings, TX_QUEUE_ACCESS (idx)); + if (thm->n_vlib_mains > vif->num_txqs) + clib_spinlock_init (&vring->lockp); } else { @@ -93,19 +91,19 @@ virtio_vring_init (vlib_main_t * vm, virtio_if_t * vif, u16 idx, u16 sz) CLIB_CACHE_LINE_BYTES); vring = vec_elt_at_index (vif->rxq_vrings, RX_QUEUE_ACCESS (idx)); } - i = sizeof (struct vring_desc) * sz; + i = sizeof (vring_desc_t) * sz; i = round_pow2 (i, CLIB_CACHE_LINE_BYTES); vring->desc = clib_mem_alloc_aligned (i, CLIB_CACHE_LINE_BYTES); clib_memset (vring->desc, 0, i); - i = sizeof (struct vring_avail) + sz * sizeof (vring->avail->ring[0]); + i = sizeof (vring_avail_t) + sz * sizeof (vring->avail->ring[0]); i = round_pow2 (i, CLIB_CACHE_LINE_BYTES); vring->avail = clib_mem_alloc_aligned (i, CLIB_CACHE_LINE_BYTES); clib_memset (vring->avail, 0, i); // tell kernel that we don't need interrupt - vring->avail->flags = VIRTIO_RING_FLAG_MASK_INT; + vring->avail->flags = VRING_AVAIL_F_NO_INTERRUPT; - i = sizeof (struct vring_used) + sz * sizeof (struct vring_used_elem); + i = sizeof (vring_used_t) + sz * sizeof (vring_used_elem_t); i = round_pow2 (i, CLIB_CACHE_LINE_BYTES); vring->used = clib_mem_alloc_aligned (i, CLIB_CACHE_LINE_BYTES); clib_memset (vring->used, 0, i); @@ -113,24 +111,17 @@ virtio_vring_init (vlib_main_t * vm, virtio_if_t * vif, u16 idx, u16 sz) vring->queue_id = idx; ASSERT (vring->buffers == 0); vec_validate_aligned (vring->buffers, sz, CLIB_CACHE_LINE_BYTES); - ASSERT (vring->indirect_buffers == 0); - vec_validate_aligned (vring->indirect_buffers, sz, CLIB_CACHE_LINE_BYTES); - if (idx % 2) + + if (idx & 1) { - u32 n_alloc = 0; - do - { - if (n_alloc < sz) - n_alloc += - vlib_buffer_alloc (vm, vring->indirect_buffers + n_alloc, - sz - n_alloc); - } - while (n_alloc != sz); + clib_memset_u32 (vring->buffers, ~0, sz); } vring->size = sz; vring->call_fd = eventfd (0, EFD_NONBLOCK | EFD_CLOEXEC); - vring->kick_fd = eventfd (0, EFD_CLOEXEC); + vring->kick_fd = eventfd (0, EFD_NONBLOCK | EFD_CLOEXEC); + virtio_log_debug (vif, "vring %u size %u call_fd %d kick_fd %d", idx, + vring->size, vring->call_fd, vring->kick_fd); t.read_function = call_read_ready; t.file_descriptor = vring->call_fd; @@ -139,31 +130,11 @@ virtio_vring_init (vlib_main_t * vm, virtio_if_t * vif, u16 idx, u16 sz) vif->dev_instance, idx); vring->call_file_index = clib_file_add (&file_main, &t); - state.index = idx; - state.num = sz; - _IOCTL (vif->fd, VHOST_SET_VRING_NUM, &state); - - addr.index = idx; - addr.flags = 0; - addr.desc_user_addr = pointer_to_uword (vring->desc); - addr.avail_user_addr = pointer_to_uword (vring->avail); - addr.used_user_addr = pointer_to_uword (vring->used); - _IOCTL (vif->fd, VHOST_SET_VRING_ADDR, &addr); - - file.index = idx; - file.fd = vring->kick_fd; - _IOCTL (vif->fd, VHOST_SET_VRING_KICK, &file); - file.fd = vring->call_fd; - _IOCTL (vif->fd, VHOST_SET_VRING_CALL, &file); - file.fd = vif->tap_fd; - _IOCTL (vif->fd, VHOST_NET_SET_BACKEND, &file); - -error: - return err; + return 0; } inline void -virtio_free_rx_buffers (vlib_main_t * vm, virtio_vring_t * vring) +virtio_free_buffers (vlib_main_t * vm, virtio_vring_t * vring) { u16 used = vring->desc_in_use; u16 last = vring->last_used_idx; @@ -188,7 +159,7 @@ virtio_vring_free_rx (vlib_main_t * vm, virtio_if_t * vif, u32 idx) close (vring->call_fd); if (vring->used) { - virtio_free_rx_buffers (vm, vring); + virtio_free_buffers (vm, vring); clib_mem_free (vring->used); } if (vring->desc) @@ -196,7 +167,6 @@ virtio_vring_free_rx (vlib_main_t * vm, virtio_if_t * vif, u32 idx) if (vring->avail) clib_mem_free (vring->avail); vec_free (vring->buffers); - vec_free (vring->indirect_buffers); return 0; } @@ -211,19 +181,57 @@ virtio_vring_free_tx (vlib_main_t * vm, virtio_if_t * vif, u32 idx) close (vring->call_fd); if (vring->used) { - virtio_free_used_desc (vm, vring); + virtio_free_buffers (vm, vring); clib_mem_free (vring->used); } if (vring->desc) clib_mem_free (vring->desc); if (vring->avail) clib_mem_free (vring->avail); - vlib_buffer_free_no_next (vm, vring->indirect_buffers, vring->size); vec_free (vring->buffers); - vec_free (vring->indirect_buffers); + gro_flow_table_free (vring->flow_table); + virtio_vring_buffering_free (vm, vring->buffering); + clib_spinlock_free (&vring->lockp); return 0; } +void +virtio_set_packet_coalesce (virtio_if_t * vif) +{ + vnet_main_t *vnm = vnet_get_main (); + vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, vif->hw_if_index); + virtio_vring_t *vring; + vif->packet_coalesce = 1; + vec_foreach (vring, vif->txq_vrings) + { + gro_flow_table_init (&vring->flow_table, + vif->type & (VIRTIO_IF_TYPE_TAP | + VIRTIO_IF_TYPE_PCI), hw->tx_node_index); + } +} + +clib_error_t * +virtio_set_packet_buffering (virtio_if_t * vif, u16 buffering_size) +{ + vnet_main_t *vnm = vnet_get_main (); + vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, vif->hw_if_index); + virtio_vring_t *vring; + clib_error_t *error = 0; + vif->packet_buffering = 1; + + vec_foreach (vring, vif->txq_vrings) + { + if ((error = + virtio_vring_buffering_init (&vring->buffering, hw->tx_node_index, + buffering_size))) + { + break; + } + } + + return error; +} + void virtio_vring_set_numa_node (vlib_main_t * vm, virtio_if_t * vif, u32 idx) { @@ -245,9 +253,9 @@ virtio_set_net_hdr_size (virtio_if_t * vif) { if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MRG_RXBUF) || vif->features & VIRTIO_FEATURE (VIRTIO_F_VERSION_1)) - vif->virtio_net_hdr_sz = sizeof (struct virtio_net_hdr_v1); + vif->virtio_net_hdr_sz = sizeof (virtio_net_hdr_v1_t); else - vif->virtio_net_hdr_sz = sizeof (struct virtio_net_hdr); + vif->virtio_net_hdr_sz = sizeof (virtio_net_hdr_t); } inline void @@ -298,18 +306,37 @@ virtio_show (vlib_main_t * vm, u32 * hw_if_indices, u8 show_descr, u32 type) vlib_cli_output (vm, " PCI Address: %U", format_vlib_pci_addr, &vif->pci_addr); } - if (type == VIRTIO_IF_TYPE_TAP) + if (type & (VIRTIO_IF_TYPE_TAP | VIRTIO_IF_TYPE_TUN)) { + u8 *str = 0; if (vif->host_if_name) vlib_cli_output (vm, " name \"%s\"", vif->host_if_name); if (vif->net_ns) vlib_cli_output (vm, " host-ns \"%s\"", vif->net_ns); - vlib_cli_output (vm, " fd %d", vif->fd); - vlib_cli_output (vm, " tap-fd %d", vif->tap_fd); - vlib_cli_output (vm, " gso-enabled %d", vif->gso_enabled); + if (vif->host_mtu_size) + vlib_cli_output (vm, " host-mtu-size \"%d\"", + vif->host_mtu_size); + if (type == VIRTIO_IF_TYPE_TAP) + vlib_cli_output (vm, " host-mac-addr: %U", + format_ethernet_address, vif->host_mac_addr); + vlib_cli_output (vm, " host-carrier-up: %u", vif->host_carrier_up); + + vec_foreach_index (i, vif->vhost_fds) + str = format (str, " %d", vif->vhost_fds[i]); + vlib_cli_output (vm, " vhost-fds%v", str); + vec_free (str); + vec_foreach_index (i, vif->tap_fds) + str = format (str, " %d", vif->tap_fds[i]); + vlib_cli_output (vm, " tap-fds%v", str); + vec_free (str); } - vlib_cli_output (vm, " Mac Address: %U", format_ethernet_address, - vif->mac_addr); + vlib_cli_output (vm, " gso-enabled %d", vif->gso_enabled); + vlib_cli_output (vm, " csum-enabled %d", vif->csum_offload_enabled); + vlib_cli_output (vm, " packet-coalesce %d", vif->packet_coalesce); + vlib_cli_output (vm, " packet-buffering %d", vif->packet_buffering); + if (type & (VIRTIO_IF_TYPE_TAP | VIRTIO_IF_TYPE_PCI)) + vlib_cli_output (vm, " Mac Address: %U", format_ethernet_address, + vif->mac_addr); vlib_cli_output (vm, " Device instance: %u", vif->dev_instance); vlib_cli_output (vm, " flags 0x%x", vif->flags); flag_entry = (struct feat_struct *) &flags_array; @@ -355,11 +382,25 @@ virtio_show (vlib_main_t * vm, u32 * hw_if_indices, u8 show_descr, u32 type) " qsz %d, last_used_idx %d, desc_next %d, desc_in_use %d", vring->size, vring->last_used_idx, vring->desc_next, vring->desc_in_use); - vlib_cli_output (vm, - " avail.flags 0x%x avail.idx %d used.flags 0x%x used.idx %d", - vring->avail->flags, vring->avail->idx, - vring->used->flags, vring->used->idx); - if (type == VIRTIO_IF_TYPE_TAP) + if (vif->is_packed) + { + vlib_cli_output (vm, + " driver_event.flags 0x%x driver_event.off_wrap %d device_event.flags 0x%x device_event.off_wrap %d", + vring->driver_event->flags, + vring->driver_event->off_wrap, + vring->device_event->flags, + vring->device_event->off_wrap); + vlib_cli_output (vm, + " avail wrap counter %d, used wrap counter %d", + vring->avail_wrap_counter, + vring->used_wrap_counter); + } + else + vlib_cli_output (vm, + " avail.flags 0x%x avail.idx %d used.flags 0x%x used.idx %d", + vring->avail->flags, vring->avail->idx, + vring->used->flags, vring->used->idx); + if (type & (VIRTIO_IF_TYPE_TAP | VIRTIO_IF_TYPE_TUN)) { vlib_cli_output (vm, " kickfd %d, callfd %d", vring->kick_fd, vring->call_fd); @@ -368,17 +409,29 @@ virtio_show (vlib_main_t * vm, u32 * hw_if_indices, u8 show_descr, u32 type) { vlib_cli_output (vm, "\n descriptor table:\n"); vlib_cli_output (vm, - " id addr len flags next user_addr\n"); + " id addr len flags next/id user_addr\n"); vlib_cli_output (vm, - " ===== ================== ===== ====== ===== ==================\n"); + " ===== ================== ===== ====== ======= ==================\n"); for (j = 0; j < vring->size; j++) { - struct vring_desc *desc = &vring->desc[j]; - vlib_cli_output (vm, - " %-5d 0x%016lx %-5d 0x%04x %-5d 0x%016lx\n", - j, desc->addr, - desc->len, - desc->flags, desc->next, desc->addr); + if (vif->is_packed) + { + vring_packed_desc_t *desc = &vring->packed_desc[j]; + vlib_cli_output (vm, + " %-5d 0x%016lx %-5d 0x%04x %-8d 0x%016lx\n", + j, desc->addr, + desc->len, + desc->flags, desc->id, desc->addr); + } + else + { + vring_desc_t *desc = &vring->desc[j]; + vlib_cli_output (vm, + " %-5d 0x%016lx %-5d 0x%04x %-8d 0x%016lx\n", + j, desc->addr, + desc->len, + desc->flags, desc->next, desc->addr); + } } } } @@ -390,30 +443,66 @@ virtio_show (vlib_main_t * vm, u32 * hw_if_indices, u8 show_descr, u32 type) " qsz %d, last_used_idx %d, desc_next %d, desc_in_use %d", vring->size, vring->last_used_idx, vring->desc_next, vring->desc_in_use); - vlib_cli_output (vm, - " avail.flags 0x%x avail.idx %d used.flags 0x%x used.idx %d", - vring->avail->flags, vring->avail->idx, - vring->used->flags, vring->used->idx); - if (type == VIRTIO_IF_TYPE_TAP) + if (vif->is_packed) + { + vlib_cli_output (vm, + " driver_event.flags 0x%x driver_event.off_wrap %d device_event.flags 0x%x device_event.off_wrap %d", + vring->driver_event->flags, + vring->driver_event->off_wrap, + vring->device_event->flags, + vring->device_event->off_wrap); + vlib_cli_output (vm, + " avail wrap counter %d, used wrap counter %d", + vring->avail_wrap_counter, + vring->used_wrap_counter); + } + else + vlib_cli_output (vm, + " avail.flags 0x%x avail.idx %d used.flags 0x%x used.idx %d", + vring->avail->flags, vring->avail->idx, + vring->used->flags, vring->used->idx); + if (type & (VIRTIO_IF_TYPE_TAP | VIRTIO_IF_TYPE_TUN)) { vlib_cli_output (vm, " kickfd %d, callfd %d", vring->kick_fd, vring->call_fd); } + if (vring->flow_table) + { + vlib_cli_output (vm, " %U", gro_flow_table_format, + vring->flow_table); + } + if (vif->packet_buffering) + { + vlib_cli_output (vm, " %U", virtio_vring_buffering_format, + vring->buffering); + } if (show_descr) { vlib_cli_output (vm, "\n descriptor table:\n"); vlib_cli_output (vm, - " id addr len flags next user_addr\n"); + " id addr len flags next/id user_addr\n"); vlib_cli_output (vm, - " ===== ================== ===== ====== ===== ==================\n"); + " ===== ================== ===== ====== ======== ==================\n"); for (j = 0; j < vring->size; j++) { - struct vring_desc *desc = &vring->desc[j]; - vlib_cli_output (vm, - " %-5d 0x%016lx %-5d 0x%04x %-5d 0x%016lx\n", - j, desc->addr, - desc->len, - desc->flags, desc->next, desc->addr); + if (vif->is_packed) + { + vring_packed_desc_t *desc = &vring->packed_desc[j]; + vlib_cli_output (vm, + " %-5d 0x%016lx %-5d 0x%04x %-8d 0x%016lx\n", + j, desc->addr, + desc->len, + desc->flags, desc->id, desc->addr); + } + else + { + vring_desc_t *desc = &vring->desc[j]; + vlib_cli_output (vm, + " %-5d 0x%016lx %-5d 0x%04x %-8d 0x%016lx\n", + j, desc->addr, + desc->len, + desc->flags, desc->next, desc->addr); + } } } } @@ -426,11 +515,27 @@ virtio_show (vlib_main_t * vm, u32 * hw_if_indices, u8 show_descr, u32 type) " qsz %d, last_used_idx %d, desc_next %d, desc_in_use %d", vring->size, vring->last_used_idx, vring->desc_next, vring->desc_in_use); - vlib_cli_output (vm, - " avail.flags 0x%x avail.idx %d used.flags 0x%x used.idx %d", - vring->avail->flags, vring->avail->idx, - vring->used->flags, vring->used->idx); - if (type == VIRTIO_IF_TYPE_TAP) + if (vif->is_packed) + { + vlib_cli_output (vm, + " driver_event.flags 0x%x driver_event.off_wrap %d device_event.flags 0x%x device_event.off_wrap %d", + vring->driver_event->flags, + vring->driver_event->off_wrap, + vring->device_event->flags, + vring->device_event->off_wrap); + vlib_cli_output (vm, + " avail wrap counter %d, used wrap counter %d", + vring->avail_wrap_counter, + vring->used_wrap_counter); + } + else + { + vlib_cli_output (vm, + " avail.flags 0x%x avail.idx %d used.flags 0x%x used.idx %d", + vring->avail->flags, vring->avail->idx, + vring->used->flags, vring->used->idx); + } + if (type & (VIRTIO_IF_TYPE_TAP | VIRTIO_IF_TYPE_TUN)) { vlib_cli_output (vm, " kickfd %d, callfd %d", vring->kick_fd, vring->call_fd); @@ -439,17 +544,29 @@ virtio_show (vlib_main_t * vm, u32 * hw_if_indices, u8 show_descr, u32 type) { vlib_cli_output (vm, "\n descriptor table:\n"); vlib_cli_output (vm, - " id addr len flags next user_addr\n"); + " id addr len flags next/id user_addr\n"); vlib_cli_output (vm, - " ===== ================== ===== ====== ===== ==================\n"); + " ===== ================== ===== ====== ======== ==================\n"); for (j = 0; j < vring->size; j++) { - struct vring_desc *desc = &vring->desc[j]; - vlib_cli_output (vm, - " %-5d 0x%016lx %-5d 0x%04x %-5d 0x%016lx\n", - j, desc->addr, - desc->len, - desc->flags, desc->next, desc->addr); + if (vif->is_packed) + { + vring_packed_desc_t *desc = &vring->packed_desc[j]; + vlib_cli_output (vm, + " %-5d 0x%016lx %-5d 0x%04x %-8d 0x%016lx\n", + j, desc->addr, + desc->len, + desc->flags, desc->id, desc->addr); + } + else + { + vring_desc_t *desc = &vring->desc[j]; + vlib_cli_output (vm, + " %-5d 0x%016lx %-5d 0x%04x %-8d 0x%016lx\n", + j, desc->addr, + desc->len, + desc->flags, desc->next, desc->addr); + } } } } @@ -458,6 +575,20 @@ virtio_show (vlib_main_t * vm, u32 * hw_if_indices, u8 show_descr, u32 type) } +static clib_error_t * +virtio_init (vlib_main_t * vm) +{ + virtio_main_t *vim = &virtio_main; + clib_error_t *error = 0; + + vim->log_default = vlib_log_register_class ("virtio", 0); + vlib_log_debug (vim->log_default, "initialized"); + + return error; +} + +VLIB_INIT_FUNCTION (virtio_init); + /* * fd.io coding-style-patch-verification: ON *