+struct virtio_ctrl_msg
+{
+ struct virtio_net_ctrl_hdr ctrl;
+ virtio_net_ctrl_ack status;
+ u8 data[1024];
+};
+
+static int
+virtio_pci_send_ctrl_msg (vlib_main_t * vm, virtio_if_t * vif,
+ struct virtio_ctrl_msg *data, u32 len)
+{
+ virtio_vring_t *vring = vif->cxq_vring;
+ virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
+ struct virtio_ctrl_msg result;
+ u32 buffer_index;
+ vlib_buffer_t *b;
+ u16 used, next, avail;
+ u16 sz = vring->size;
+ u16 mask = sz - 1;
+
+ used = vring->desc_in_use;
+ next = vring->desc_next;
+ avail = vring->avail->idx;
+ struct vring_desc *d = &vring->desc[next];
+
+ if (vlib_buffer_alloc (vm, &buffer_index, 1))
+ b = vlib_get_buffer (vm, buffer_index);
+ else
+ return VIRTIO_NET_ERR;
+ /*
+ * current_data may not be initialized with 0 and may contain
+ * previous offset.
+ */
+ b->current_data = 0;
+ clib_memcpy (vlib_buffer_get_current (b), data,
+ sizeof (struct virtio_ctrl_msg));
+ d->flags = VRING_DESC_F_NEXT;
+ d->addr = vlib_buffer_get_current_pa (vm, b);
+ d->len = sizeof (struct virtio_net_ctrl_hdr);
+ vring->avail->ring[avail & mask] = next;
+ avail++;
+ next = (next + 1) & mask;
+ d->next = next;
+ used++;
+
+ d = &vring->desc[next];
+ d->flags = VRING_DESC_F_NEXT;
+ d->addr = vlib_buffer_get_current_pa (vm, b) +
+ STRUCT_OFFSET_OF (struct virtio_ctrl_msg, data);
+ d->len = len;
+ next = (next + 1) & mask;
+ d->next = next;
+ used++;
+
+ d = &vring->desc[next];
+ d->flags = VRING_DESC_F_WRITE;
+ d->addr = vlib_buffer_get_current_pa (vm, b) +
+ STRUCT_OFFSET_OF (struct virtio_ctrl_msg, status);
+ d->len = sizeof (data->status);
+ next = (next + 1) & mask;
+ used++;
+
+ CLIB_MEMORY_STORE_BARRIER ();
+ vring->avail->idx = avail;
+ vring->desc_next = next;
+ vring->desc_in_use = used;
+
+ if ((vring->used->flags & VIRTIO_RING_FLAG_MASK_INT) == 0)
+ {
+ virtio_kick (vm, vring, vif);
+ }
+
+ u16 last = vring->last_used_idx, n_left = 0;
+ n_left = vring->used->idx - last;
+
+ while (n_left)
+ {
+ struct vring_used_elem *e = &vring->used->ring[last & mask];
+ u16 slot = e->id;
+
+ d = &vring->desc[slot];
+ while (d->flags & VRING_DESC_F_NEXT)
+ {
+ used--;
+ slot = d->next;
+ d = &vring->desc[slot];
+ }
+ used--;
+ last++;
+ n_left--;
+ }
+ vring->desc_in_use = used;
+ vring->last_used_idx = last;
+
+ CLIB_MEMORY_BARRIER ();
+ clib_memcpy (&result, vlib_buffer_get_current (b),
+ sizeof (struct virtio_ctrl_msg));
+ virtio_log_debug (vif, "ctrl-queue: status %u", result.status);
+ status = result.status;
+ vlib_buffer_free (vm, &buffer_index, 1);
+ return status;
+}
+
+static int
+virtio_pci_disable_offload (vlib_main_t * vm, virtio_if_t * vif)
+{
+ struct virtio_ctrl_msg offload_hdr;
+ virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
+
+ offload_hdr.ctrl.class = VIRTIO_NET_CTRL_GUEST_OFFLOADS;
+ offload_hdr.ctrl.cmd = VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET;
+ offload_hdr.status = VIRTIO_NET_ERR;
+ u64 offloads = 0ULL;
+ clib_memcpy (offload_hdr.data, &offloads, sizeof (offloads));
+
+ status =
+ virtio_pci_send_ctrl_msg (vm, vif, &offload_hdr, sizeof (offloads));
+ virtio_log_debug (vif, "disable offloads");
+ vif->remote_features = virtio_pci_legacy_get_host_features (vm, vif);
+ virtio_pci_legacy_get_guest_features (vm, vif);
+ return status;
+}
+
+static int
+virtio_pci_enable_checksum_offload (vlib_main_t * vm, virtio_if_t * vif)
+{
+ struct virtio_ctrl_msg csum_offload_hdr;
+ virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
+
+ csum_offload_hdr.ctrl.class = VIRTIO_NET_CTRL_GUEST_OFFLOADS;
+ csum_offload_hdr.ctrl.cmd = VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET;
+ csum_offload_hdr.status = VIRTIO_NET_ERR;
+ u64 offloads = 0ULL;
+ offloads |= VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_CSUM);
+ clib_memcpy (csum_offload_hdr.data, &offloads, sizeof (offloads));
+
+ status =
+ virtio_pci_send_ctrl_msg (vm, vif, &csum_offload_hdr, sizeof (offloads));
+ virtio_log_debug (vif, "enable checksum offload");
+ vif->remote_features = virtio_pci_legacy_get_host_features (vm, vif);
+ virtio_pci_legacy_get_guest_features (vm, vif);
+ return status;
+}
+
+static int
+virtio_pci_enable_gso (vlib_main_t * vm, virtio_if_t * vif)
+{
+ struct virtio_ctrl_msg gso_hdr;
+ virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
+
+ gso_hdr.ctrl.class = VIRTIO_NET_CTRL_GUEST_OFFLOADS;
+ gso_hdr.ctrl.cmd = VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET;
+ gso_hdr.status = VIRTIO_NET_ERR;
+ u64 offloads = VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_CSUM)
+ | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO4)
+ | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO6);
+ clib_memcpy (gso_hdr.data, &offloads, sizeof (offloads));
+
+ status = virtio_pci_send_ctrl_msg (vm, vif, &gso_hdr, sizeof (offloads));
+ virtio_log_debug (vif, "enable gso");
+ vif->remote_features = virtio_pci_legacy_get_host_features (vm, vif);
+ virtio_pci_legacy_get_guest_features (vm, vif);
+ return status;
+}
+
+static int
+virtio_pci_offloads (vlib_main_t * vm, virtio_if_t * vif, int gso_enabled,
+ int csum_offload_enabled)
+{
+ vnet_main_t *vnm = vnet_get_main ();
+ vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, vif->hw_if_index);
+
+ if ((vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ)) &&
+ (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)))
+ {
+ if (gso_enabled
+ && (vif->features & (VIRTIO_FEATURE (VIRTIO_NET_F_HOST_TSO4) |
+ VIRTIO_FEATURE (VIRTIO_NET_F_HOST_TSO6))))
+ {
+ if (virtio_pci_enable_gso (vm, vif))
+ {
+ virtio_log_warning (vif, "gso is not enabled");
+ }
+ else
+ {
+ vif->gso_enabled = 1;
+ vif->csum_offload_enabled = 0;
+ hw->flags |= VNET_HW_INTERFACE_FLAG_SUPPORTS_GSO |
+ VNET_HW_INTERFACE_FLAG_SUPPORTS_TX_L4_CKSUM_OFFLOAD;
+ }
+ }
+ else if (csum_offload_enabled
+ && (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CSUM)))
+ {
+ if (virtio_pci_enable_checksum_offload (vm, vif))
+ {
+ virtio_log_warning (vif, "checksum offload is not enabled");
+ }
+ else
+ {
+ vif->csum_offload_enabled = 1;
+ vif->gso_enabled = 0;
+ hw->flags &= ~VNET_HW_INTERFACE_FLAG_SUPPORTS_GSO;
+ hw->flags |=
+ VNET_HW_INTERFACE_FLAG_SUPPORTS_TX_L4_CKSUM_OFFLOAD;
+ }
+ }
+ else
+ {
+ if (virtio_pci_disable_offload (vm, vif))
+ {
+ virtio_log_warning (vif, "offloads are not disabled");
+ }
+ else
+ {
+ vif->csum_offload_enabled = 0;
+ vif->gso_enabled = 0;
+ hw->flags &= ~(VNET_HW_INTERFACE_FLAG_SUPPORTS_GSO |
+ VNET_HW_INTERFACE_FLAG_SUPPORTS_TX_L4_CKSUM_OFFLOAD);
+ }
+ }
+ }
+
+ return 0;
+}
+
+static int
+virtio_pci_enable_multiqueue (vlib_main_t * vm, virtio_if_t * vif,
+ u16 num_queues)
+{
+ struct virtio_ctrl_msg mq_hdr;
+ virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
+
+ mq_hdr.ctrl.class = VIRTIO_NET_CTRL_MQ;
+ mq_hdr.ctrl.cmd = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET;
+ mq_hdr.status = VIRTIO_NET_ERR;
+ clib_memcpy (mq_hdr.data, &num_queues, sizeof (num_queues));
+
+ status = virtio_pci_send_ctrl_msg (vm, vif, &mq_hdr, sizeof (num_queues));
+ virtio_log_debug (vif, "multi-queue enable %u queues", num_queues);
+ return status;
+}
+