+static void
+virtio_free_used_device_desc_packed (vlib_main_t *vm,
+ vnet_virtio_vring_t *vring,
+ uword node_index)
+{
+ vnet_virtio_vring_packed_desc_t *d;
+ u16 sz = vring->queue_size;
+ u16 last = vring->last_used_idx;
+ u16 n_buffers = 0, start;
+ u16 flags;
+
+ if (vring->desc_in_use == 0)
+ return;
+
+ d = &vring->packed_desc[last];
+ flags = d->flags;
+ start = d->id;
+
+ while ((flags & VRING_DESC_F_AVAIL) == (vring->used_wrap_counter << 7) &&
+ (flags & VRING_DESC_F_USED) == (vring->used_wrap_counter << 15))
+ {
+ last++;
+ n_buffers++;
+
+ if (last >= sz)
+ {
+ last = 0;
+ vring->used_wrap_counter ^= 1;
+ }
+ d = &vring->packed_desc[last];
+ flags = d->flags;
+ }
+
+ if (n_buffers)
+ {
+ vlib_buffer_free_from_ring (vm, vring->buffers, start, sz, n_buffers);
+ virtio_memset_ring_u32 (vring->buffers, start, sz, n_buffers);
+ vring->desc_in_use -= n_buffers;
+ vring->last_used_idx = last;
+ }
+}
+
+static void
+virtio_free_used_device_desc (vlib_main_t *vm, vnet_virtio_vring_t *vring,
+ uword node_index, int packed)
+{
+ if (packed)
+ virtio_free_used_device_desc_packed (vm, vring, node_index);
+ else
+ virtio_free_used_device_desc_split (vm, vring, node_index);
+
+}
+
+static void
+set_checksum_offsets (vlib_buffer_t *b, vnet_virtio_net_hdr_v1_t *hdr,
+ const int is_l2)
+{
+ vnet_buffer_oflags_t oflags = vnet_buffer (b)->oflags;
+ i16 l4_hdr_offset = vnet_buffer (b)->l4_hdr_offset - b->current_data;
+ if (b->flags & VNET_BUFFER_F_IS_IP4)
+ {
+ ip4_header_t *ip4;
+ hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
+ hdr->csum_start = l4_hdr_offset; // 0x22;
+
+ /*
+ * virtio devices do not support IP4 checksum offload. So driver takes
+ * care of it while doing tx.
+ */
+ ip4 = (ip4_header_t *) (b->data + vnet_buffer (b)->l3_hdr_offset);
+ if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
+ ip4->checksum = ip4_header_checksum (ip4);
+
+ /*
+ * virtio devices assume the l4 header is set to the checksum of the
+ * l3 pseudo-header, so we compute it before tx-ing
+ */
+ if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
+ {
+ tcp_header_t *tcp =
+ (tcp_header_t *) (b->data + vnet_buffer (b)->l4_hdr_offset);
+ tcp->checksum = ip4_pseudo_header_cksum (ip4);
+ hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
+ }
+ else if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
+ {
+ udp_header_t *udp =
+ (udp_header_t *) (b->data + vnet_buffer (b)->l4_hdr_offset);
+ udp->checksum = ip4_pseudo_header_cksum (ip4);
+ hdr->csum_offset = STRUCT_OFFSET_OF (udp_header_t, checksum);
+ }
+ }
+ else if (b->flags & VNET_BUFFER_F_IS_IP6)
+ {
+ ip6_header_t *ip6;
+ hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
+ hdr->csum_start = l4_hdr_offset; // 0x36;
+ ip6 = (ip6_header_t *) (b->data + vnet_buffer (b)->l3_hdr_offset);
+
+ /*
+ * virtio devices assume the l4 header is set to the checksum of the
+ * l3 pseudo-header, so we compute it before tx-ing
+ */
+ if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
+ {
+ tcp_header_t *tcp =
+ (tcp_header_t *) (b->data + vnet_buffer (b)->l4_hdr_offset);
+ tcp->checksum = ip6_pseudo_header_cksum (ip6);
+ hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
+ }
+ else if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
+ {
+ udp_header_t *udp =
+ (udp_header_t *) (b->data + vnet_buffer (b)->l4_hdr_offset);
+ udp->checksum = ip6_pseudo_header_cksum (ip6);
+ hdr->csum_offset = STRUCT_OFFSET_OF (udp_header_t, checksum);
+ }
+ }
+}
+
+static void
+set_gso_offsets (vlib_buffer_t *b, vnet_virtio_net_hdr_v1_t *hdr,
+ const int is_l2)
+{
+ vnet_buffer_oflags_t oflags = vnet_buffer (b)->oflags;
+ i16 l4_hdr_offset = vnet_buffer (b)->l4_hdr_offset - b->current_data;
+
+ if (b->flags & VNET_BUFFER_F_IS_IP4)
+ {
+ ip4_header_t *ip4;
+ hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
+ hdr->gso_size = vnet_buffer2 (b)->gso_size;
+ hdr->hdr_len = l4_hdr_offset + vnet_buffer2 (b)->gso_l4_hdr_sz;
+ hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
+ hdr->csum_start = l4_hdr_offset; // 0x22;
+ hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
+ ip4 = (ip4_header_t *) (b->data + vnet_buffer (b)->l3_hdr_offset);
+ /*
+ * virtio devices do not support IP4 checksum offload. So driver takes care
+ * of it while doing tx.
+ */
+ if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
+ ip4->checksum = ip4_header_checksum (ip4);
+ }
+ else if (b->flags & VNET_BUFFER_F_IS_IP6)
+ {
+ hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
+ hdr->gso_size = vnet_buffer2 (b)->gso_size;
+ hdr->hdr_len = l4_hdr_offset + vnet_buffer2 (b)->gso_l4_hdr_sz;
+ hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
+ hdr->csum_start = l4_hdr_offset; // 0x36;
+ hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
+ }
+}
+
+static u16
+add_buffer_to_slot (vlib_main_t *vm, vlib_node_runtime_t *node,
+ virtio_if_t *vif, vnet_virtio_vring_t *vring, u32 bi,
+ u16 free_desc_count, u16 avail, u16 next, u16 mask,
+ int hdr_sz, int do_gso, int csum_offload, int is_pci,
+ int is_tun, int is_indirect, int is_any_layout)