#include <vlib/vlib.h>
#include <vlib/unix/unix.h>
+#include <vnet/vnet.h>
#include <vnet/ethernet/ethernet.h>
#include <vnet/gso/gso.h>
#include <vnet/ip/ip4_packet.h>
_(NO_FREE_SLOTS, "no free tx slots") \
_(TRUNC_PACKET, "packet > buffer size -- truncated in tx ring") \
_(PENDING_MSGS, "pending msgs in tx ring") \
-_(NO_TX_QUEUES, "no tx queues") \
-_(OUT_OF_ORDER, "out-of-order buffers in used ring")
+_(INDIRECT_DESC_ALLOC_FAILED, "indirect descriptor allocation failed - packet drop") \
+_(OUT_OF_ORDER, "out-of-order buffers in used ring") \
+_(GSO_PACKET_DROP, "gso disabled on itf -- gso packet drop") \
+_(CSUM_OFFLOAD_PACKET_DROP, "checksum offload disabled on itf -- csum offload packet drop")
typedef enum
{
return s;
}
+static_always_inline void
+virtio_interface_drop_inline (vlib_main_t * vm, uword node_index,
+ u32 * buffers, u16 n,
+ virtio_tx_func_error_t error)
+{
+ vlib_error_count (vm, node_index, error, n);
+ vlib_buffer_free (vm, buffers, n);
+}
+
static_always_inline void
virtio_memset_ring_u32 (u32 * ring, u32 start, u32 ring_size, u32 n_buffers)
{
n_left--;
last++;
n_buffers++;
+ struct vring_desc *d = &vring->desc[e->id];
+ u16 next;
+ while (d->flags & VRING_DESC_F_NEXT)
+ {
+ n_buffers++;
+ next = d->next;
+ d = &vring->desc[next];
+ }
if (n_left == 0)
break;
e = &vring->used->ring[last & mask];
}
static_always_inline void
-set_checksum_offsets (vlib_main_t * vm, virtio_if_t * vif, vlib_buffer_t * b,
- struct virtio_net_hdr_v1 *hdr)
+set_checksum_offsets (vlib_buffer_t * b, struct virtio_net_hdr_v1 *hdr)
{
if (b->flags & VNET_BUFFER_F_IS_IP4)
{
hdr->csum_start = gho.l4_hdr_offset; // 0x22;
if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
{
- tcp_header_t *tcp =
- (tcp_header_t *) (vlib_buffer_get_current (b) +
- gho.l4_hdr_offset);
- tcp->checksum = 0;
hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
}
else if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
{
- udp_header_t *udp =
- (udp_header_t *) (vlib_buffer_get_current (b) +
- gho.l4_hdr_offset);
- udp->checksum = 0;
hdr->csum_offset = STRUCT_OFFSET_OF (udp_header_t, checksum);
}
hdr->csum_start = gho.l4_hdr_offset; // 0x36;
if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
{
- tcp_header_t *tcp =
- (tcp_header_t *) (vlib_buffer_get_current (b) +
- gho.l4_hdr_offset);
- tcp->checksum = 0;
hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
}
else if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
{
- udp_header_t *udp =
- (udp_header_t *) (vlib_buffer_get_current (b) +
- gho.l4_hdr_offset);
- udp->checksum = 0;
hdr->csum_offset = STRUCT_OFFSET_OF (udp_header_t, checksum);
}
}
}
+static_always_inline void
+set_gso_offsets (vlib_buffer_t * b, struct virtio_net_hdr_v1 *hdr)
+{
+ if (b->flags & VNET_BUFFER_F_IS_IP4)
+ {
+ ip4_header_t *ip4;
+ gso_header_offset_t gho = vnet_gso_header_offset_parser (b, 0);
+ hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
+ hdr->gso_size = vnet_buffer2 (b)->gso_size;
+ hdr->hdr_len = gho.l4_hdr_offset + gho.l4_hdr_sz;
+ hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
+ hdr->csum_start = gho.l4_hdr_offset; // 0x22;
+ hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
+ ip4 =
+ (ip4_header_t *) (vlib_buffer_get_current (b) + gho.l3_hdr_offset);
+ /*
+ * virtio devices do not support IP4 checksum offload. So driver takes care
+ * of it while doing tx.
+ */
+ if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
+ ip4->checksum = ip4_header_checksum (ip4);
+ }
+ else if (b->flags & VNET_BUFFER_F_IS_IP6)
+ {
+ gso_header_offset_t gho = vnet_gso_header_offset_parser (b, 1);
+ hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
+ hdr->gso_size = vnet_buffer2 (b)->gso_size;
+ hdr->hdr_len = gho.l4_hdr_offset + gho.l4_hdr_sz;
+ hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
+ hdr->csum_start = gho.l4_hdr_offset; // 0x36;
+ hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
+ }
+}
+
static_always_inline u16
add_buffer_to_slot (vlib_main_t * vm, virtio_if_t * vif,
- virtio_vring_t * vring, u32 bi, u16 avail, u16 next,
- u16 mask, int do_gso, int csum_offload)
+ virtio_vring_t * vring, u32 bi, u16 free_desc_count,
+ u16 avail, u16 next, u16 mask, int do_gso,
+ int csum_offload, uword node_index)
{
u16 n_added = 0;
int hdr_sz = vif->virtio_net_hdr_sz;
clib_memset (hdr, 0, hdr_sz);
- if (do_gso && (b->flags & VNET_BUFFER_F_GSO))
+ if (b->flags & VNET_BUFFER_F_GSO)
{
- if (b->flags & VNET_BUFFER_F_IS_IP4)
+ if (do_gso)
+ set_gso_offsets (b, hdr);
+ else
{
- ip4_header_t *ip4;
- tcp_header_t *tcp;
- gso_header_offset_t gho = vnet_gso_header_offset_parser (b, 0);
- hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
- hdr->gso_size = vnet_buffer2 (b)->gso_size;
- hdr->hdr_len = gho.l4_hdr_offset + gho.l4_hdr_sz;
- tcp =
- (tcp_header_t *) (vlib_buffer_get_current (b) +
- gho.l4_hdr_offset);
- tcp->checksum = 0;
- hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
- hdr->csum_start = gho.l4_hdr_offset; // 0x22;
- hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
- ip4 =
- (ip4_header_t *) (vlib_buffer_get_current (b) +
- gho.l3_hdr_offset);
- /*
- * virtio devices do not support IP4 checksum offload. So driver takes care
- * of it while doing tx.
- */
- if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
- ip4->checksum = ip4_header_checksum (ip4);
- }
- else if (b->flags & VNET_BUFFER_F_IS_IP6)
- {
- tcp_header_t *tcp;
- gso_header_offset_t gho = vnet_gso_header_offset_parser (b, 1);
- hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
- hdr->gso_size = vnet_buffer2 (b)->gso_size;
- hdr->hdr_len = gho.l4_hdr_offset + gho.l4_hdr_sz;
- hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
- tcp =
- (tcp_header_t *) (vlib_buffer_get_current (b) +
- gho.l4_hdr_offset);
- tcp->checksum = 0;
- hdr->csum_start = gho.l4_hdr_offset; // 0x36;
- hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
+ virtio_interface_drop_inline (vm, node_index, &bi, 1,
+ VIRTIO_TX_ERROR_GSO_PACKET_DROP);
+ return n_added;
}
}
- else if (csum_offload
- && (b->flags & (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)))
+ else if (b->flags & (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
+ VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
{
- set_checksum_offsets (vm, vif, b, hdr);
+ if (csum_offload)
+ set_checksum_offsets (b, hdr);
+ else
+ {
+ virtio_interface_drop_inline (vm, node_index, &bi, 1,
+ VIRTIO_TX_ERROR_CSUM_OFFLOAD_PACKET_DROP);
+ return n_added;
+ }
}
if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
d->len = b->current_length + hdr_sz;
d->flags = 0;
}
- else
+ else if (vif->features & VIRTIO_FEATURE (VIRTIO_RING_F_INDIRECT_DESC))
{
/*
* We are using single vlib_buffer_t for indirect descriptor(s)
*/
u32 indirect_buffer = 0;
if (PREDICT_FALSE (vlib_buffer_alloc (vm, &indirect_buffer, 1) == 0))
- return n_added;
+ {
+ virtio_interface_drop_inline (vm, node_index, &bi, 1,
+ VIRTIO_TX_ERROR_INDIRECT_DESC_ALLOC_FAILED);
+ return n_added;
+ }
vlib_buffer_t *indirect_desc = vlib_get_buffer (vm, indirect_buffer);
indirect_desc->current_data = 0;
id->len = b->current_length;
}
}
- else /* VIRTIO_IF_TYPE_TAP */
+ else /* VIRTIO_IF_TYPE_[TAP | TUN] */
{
d->addr = pointer_to_uword (id);
/* first buffer in chain */
d->len = count * sizeof (struct vring_desc);
d->flags = VRING_DESC_F_INDIRECT;
}
+ else if (vif->type == VIRTIO_IF_TYPE_PCI)
+ {
+ u16 count = next;
+ vlib_buffer_t *b_temp = b;
+ u16 n_buffers_in_chain = 1;
+
+ /*
+ * Check the length of the chain for the required number of
+ * descriptors. Return from here, retry to get more descriptors,
+ * if chain length is greater than available descriptors.
+ */
+ while (b_temp->flags & VLIB_BUFFER_NEXT_PRESENT)
+ {
+ n_buffers_in_chain++;
+ b_temp = vlib_get_buffer (vm, b_temp->next_buffer);
+ }
+
+ if (n_buffers_in_chain > free_desc_count)
+ return n_buffers_in_chain;
+
+ d->addr = vlib_buffer_get_current_pa (vm, b) - hdr_sz;
+ d->len = b->current_length + hdr_sz;
+
+ while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
+ {
+ d->flags = VRING_DESC_F_NEXT;
+ vring->buffers[count] = bi;
+ b->flags &=
+ ~(VLIB_BUFFER_NEXT_PRESENT | VLIB_BUFFER_TOTAL_LENGTH_VALID);
+ bi = b->next_buffer;
+ b->next_buffer = 0;
+ n_added++;
+ count = (count + 1) & mask;
+ d->next = count;
+ d = &vring->desc[count];
+ b = vlib_get_buffer (vm, bi);
+ d->addr = vlib_buffer_get_current_pa (vm, b);
+ d->len = b->current_length;
+ }
+ d->flags = 0;
+ vring->buffers[count] = bi;
+ vring->avail->ring[avail & mask] = next;
+ n_added++;
+ return n_added;
+ }
+ else
+ {
+ ASSERT (0);
+ }
vring->buffers[next] = bi;
vring->avail->ring[avail & mask] = next;
n_added++;
{
u16 n_added = 0;
n_added =
- add_buffer_to_slot (vm, vif, vring, buffers[0], avail, next, mask,
- do_gso, csum_offload);
- if (!n_added)
+ add_buffer_to_slot (vm, vif, vring, buffers[0], free_desc_count,
+ avail, next, mask, do_gso, csum_offload,
+ node->node_index);
+
+ if (PREDICT_FALSE (n_added == 0))
+ {
+ buffers++;
+ n_left--;
+ continue;
+ }
+ else if (PREDICT_FALSE (n_added > free_desc_count))
break;
- avail += n_added;
+
+ avail++;
next = (next + n_added) & mask;
used += n_added;
buffers++;
n_left--;
- free_desc_count--;
+ free_desc_count -= n_added;
}
if (n_left != frame->n_vectors)
if (retry_count--)
goto retry;
- vlib_error_count (vm, node->node_index, VIRTIO_TX_ERROR_NO_FREE_SLOTS,
- n_left);
- vlib_buffer_free (vm, buffers, n_left);
+ virtio_interface_drop_inline (vm, node->node_index, buffers, n_left,
+ VIRTIO_TX_ERROR_NO_FREE_SLOTS);
}
clib_spinlock_unlock_if_init (&vring->lockp);
.subif_add_del_function = virtio_subif_add_del_function,
.rx_mode_change_function = virtio_interface_rx_mode_change,
};
+
/* *INDENT-ON* */
/*
Code Review / vpp.git / blobdiff