#ifndef _VNET_DEVICES_VIRTIO_VIRTIO_H_
#define _VNET_DEVICES_VIRTIO_VIRTIO_H_
-#include <linux/virtio_config.h>
-#include <linux/virtio_net.h>
-#include <linux/virtio_pci.h>
-#include <linux/virtio_ring.h>
-
-#define foreach_virtio_net_features \
- _ (VIRTIO_NET_F_CSUM, 0) /* Host handles pkts w/ partial csum */ \
- _ (VIRTIO_NET_F_GUEST_CSUM, 1) /* Guest handles pkts w/ partial csum */ \
- _ (VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, 2) /* Dynamic offload configuration. */ \
- _ (VIRTIO_NET_F_MTU, 3) /* Initial MTU advice. */ \
- _ (VIRTIO_NET_F_MAC, 5) /* Host has given MAC address. */ \
- _ (VIRTIO_NET_F_GSO, 6) /* Host handles pkts w/ any GSO. */ \
- _ (VIRTIO_NET_F_GUEST_TSO4, 7) /* Guest can handle TSOv4 in. */ \
- _ (VIRTIO_NET_F_GUEST_TSO6, 8) /* Guest can handle TSOv6 in. */ \
- _ (VIRTIO_NET_F_GUEST_ECN, 9) /* Guest can handle TSO[6] w/ ECN in. */ \
- _ (VIRTIO_NET_F_GUEST_UFO, 10) /* Guest can handle UFO in. */ \
- _ (VIRTIO_NET_F_HOST_TSO4, 11) /* Host can handle TSOv4 in. */ \
- _ (VIRTIO_NET_F_HOST_TSO6, 12) /* Host can handle TSOv6 in. */ \
- _ (VIRTIO_NET_F_HOST_ECN, 13) /* Host can handle TSO[6] w/ ECN in. */ \
- _ (VIRTIO_NET_F_HOST_UFO, 14) /* Host can handle UFO in. */ \
- _ (VIRTIO_NET_F_MRG_RXBUF, 15) /* Host can merge receive buffers. */ \
- _ (VIRTIO_NET_F_STATUS, 16) /* virtio_net_config.status available */ \
- _ (VIRTIO_NET_F_CTRL_VQ, 17) /* Control channel available */ \
- _ (VIRTIO_NET_F_CTRL_RX, 18) /* Control channel RX mode support */ \
- _ (VIRTIO_NET_F_CTRL_VLAN, 19) /* Control channel VLAN filtering */ \
- _ (VIRTIO_NET_F_CTRL_RX_EXTRA, 20) /* Extra RX mode control support */ \
- _ (VIRTIO_NET_F_GUEST_ANNOUNCE, 21) /* Guest can announce device on the network */ \
- _ (VIRTIO_NET_F_MQ, 22) /* Device supports Receive Flow Steering */ \
- _ (VIRTIO_NET_F_CTRL_MAC_ADDR, 23) /* Set MAC address */ \
- _ (VIRTIO_F_NOTIFY_ON_EMPTY, 24) \
- _ (VHOST_F_LOG_ALL, 26) /* Log all write descriptors */ \
- _ (VIRTIO_F_ANY_LAYOUT, 27) /* Can the device handle any descriptor layout */ \
- _ (VIRTIO_RING_F_INDIRECT_DESC, 28) /* Support indirect buffer descriptors */ \
- _ (VIRTIO_RING_F_EVENT_IDX, 29) /* The Guest publishes the used index for which it expects an interrupt \
- * at the end of the avail ring. Host should ignore the avail->flags field. */ \
-/* The Host publishes the avail index for which it expects a kick \
- * at the end of the used ring. Guest should ignore the used->flags field. */ \
- _ (VHOST_USER_F_PROTOCOL_FEATURES, 30) \
- _ (VIRTIO_F_VERSION_1, 32)
-
+#include <vnet/devices/virtio/virtio_std.h>
+#include <vnet/devices/virtio/vhost_std.h>
+#include <vnet/devices/virtio/virtio_buffering.h>
+#include <vnet/gso/gro.h>
+#include <vnet/interface.h>
#define foreach_virtio_if_flag \
_(0, ADMIN_UP, "admin-up") \
#undef _
} virtio_if_flag_t;
-#define VIRTIO_NUM_RX_DESC 256
-#define VIRTIO_NUM_TX_DESC 256
-
-#define VIRTIO_FEATURE(X) (1ULL << X)
-
#define TX_QUEUE(X) ((X*2) + 1)
#define RX_QUEUE(X) (X*2)
#define TX_QUEUE_ACCESS(X) (X/2)
#define RX_QUEUE_ACCESS(X) (X/2)
+#define VIRTIO_NUM_RX_DESC 256
+#define VIRTIO_NUM_TX_DESC 256
+
#define foreach_virtio_if_types \
- _ (TAP, 1) \
- _ (TUN, 2) \
- _ (PCI, 3)
+ _ (TAP, 0) \
+ _ (TUN, 1) \
+ _ (PCI, 2)
typedef enum
{
#define _(a, b) VIRTIO_IF_TYPE_##a = (1 << b),
foreach_virtio_if_types
#undef _
- VIRTIO_IF_N_TYPES = (1 << 4),
+ VIRTIO_IF_N_TYPES = (1 << 3),
} virtio_if_type_t;
-
-typedef struct
-{
- u8 mac[6];
- u16 status;
- u16 max_virtqueue_pairs;
- u16 mtu;
-} virtio_net_config_t;
-
#define VIRTIO_RING_FLAG_MASK_INT 1
+#define VIRTIO_EVENT_START_TIMER 1
+#define VIRTIO_EVENT_STOP_TIMER 2
+
typedef struct
{
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
- struct vring_desc *desc;
- struct vring_used *used;
- struct vring_avail *avail;
clib_spinlock_t lockp;
+ union
+ {
+ struct
+ {
+ vnet_virtio_vring_desc_t *desc;
+ vnet_virtio_vring_used_t *used;
+ vnet_virtio_vring_avail_t *avail;
+ };
+ struct
+ {
+ vnet_virtio_vring_packed_desc_t *packed_desc;
+ vnet_virtio_vring_desc_event_t *driver_event;
+ vnet_virtio_vring_desc_event_t *device_event;
+ };
+ };
+ u32 *buffers;
+ u16 queue_size;
+ u16 queue_id;
+ u32 queue_index;
u16 desc_in_use;
u16 desc_next;
- int kick_fd;
- int call_fd;
- u8 buffer_pool_index;
- u16 size;
- u16 queue_id;
-#define VRING_TX_OUT_OF_ORDER 1
- u16 flags;
- u32 call_file_index;
- u32 *buffers;
u16 last_used_idx;
u16 last_kick_avail_idx;
-} virtio_vring_t;
+ union
+ {
+ struct
+ {
+ int kick_fd;
+ int call_fd;
+ u32 call_file_index;
+ };
+ struct
+ {
+ u16 avail_wrap_counter;
+ u16 used_wrap_counter;
+ u16 queue_notify_offset;
+ };
+ };
+#define VRING_TX_OUT_OF_ORDER 1
+#define VRING_TX_SCHEDULED 2
+ u16 flags;
+ u8 buffer_pool_index;
+ vnet_hw_if_rx_mode mode;
+ virtio_vring_buffering_t *buffering;
+ gro_flow_table_t *flow_table;
+} vnet_virtio_vring_t;
typedef union
{
u32 as_u32;
} pci_addr_t;
+/* forward declaration */
+typedef struct _virtio_pci_func virtio_pci_func_t;
+
typedef struct
{
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+ u64 features;
u32 flags;
+ u32 per_interface_next_index;
+ u16 num_rxqs;
+ u16 num_txqs;
+ vnet_virtio_vring_t *rxq_vrings;
+ vnet_virtio_vring_t *txq_vrings;
+ int gso_enabled;
+ int csum_offload_enabled;
+ union
+ {
+ int *tap_fds;
+ struct
+ {
+ u32 pci_dev_handle;
+ u32 msix_enabled;
+ };
+ };
+ u16 virtio_net_hdr_sz;
+ virtio_if_type_t type;
- u32 dev_instance;
u32 hw_if_index;
u32 sw_if_index;
+
+ CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
+ int packet_coalesce;
+ int packet_buffering;
+ u32 dev_instance;
u32 numa_node;
- u16 virtio_net_hdr_sz;
- virtio_if_type_t type;
- union
- {
- u32 id;
- pci_addr_t pci_addr;
- };
- u32 per_interface_next_index;
- int *vhost_fds;
- int *tap_fds;
- u32 msix_enabled;
- u32 pci_dev_handle;
- virtio_vring_t *rxq_vrings;
- virtio_vring_t *txq_vrings;
- u64 features, remote_features;
+ u64 remote_features;
/* error */
clib_error_t *error;
- u8 support_int_mode; /* support interrupt mode */
- u16 max_queue_pairs;
- u16 num_rxqs;
- u16 num_txqs;
- u8 status;
- u8 mac_addr[6];
- u8 *host_if_name;
- u8 *net_ns;
- u8 *host_bridge;
- u8 host_mac_addr[6];
- ip4_address_t host_ip4_addr;
- u8 host_ip4_prefix_len;
- ip6_address_t host_ip6_addr;
- u8 host_ip6_prefix_len;
- u32 host_mtu_size;
- int gso_enabled;
- int csum_offload_enabled;
- int ifindex;
- virtio_vring_t *cxq_vring;
+ union
+ {
+ struct
+ {
+ u32 mac_addr32;
+ u16 mac_addr16;
+ };
+ u8 mac_addr[6];
+ };
+ union
+ {
+ struct /* tun/tap interface */
+ {
+ ip6_address_t host_ip6_addr;
+ int *vhost_fds;
+ u8 *host_if_name;
+ u8 *net_ns;
+ u8 *host_bridge;
+ u8 host_mac_addr[6];
+ u32 id;
+ u32 host_mtu_size;
+ u32 tap_flags;
+ int ifindex;
+ ip4_address_t host_ip4_addr;
+ u8 host_ip4_prefix_len;
+ u8 host_ip6_prefix_len;
+ u8 host_carrier_up; /* host tun/tap driver link carrier state */
+ };
+ struct /* native virtio */
+ {
+ void *bar;
+ vnet_virtio_vring_t *cxq_vring;
+ pci_addr_t pci_addr;
+ u32 bar_id;
+ u32 notify_off_multiplier;
+ u32 is_modern;
+ u16 common_offset;
+ u16 notify_offset;
+ u16 device_offset;
+ u16 isr_offset;
+ u16 max_queue_pairs;
+ u16 msix_table_size;
+ u8 support_int_mode; /* support interrupt mode */
+ u8 status;
+ };
+ };
+ const virtio_pci_func_t *virtio_pci_func;
+ int is_packed;
} virtio_if_t;
typedef struct
{
+ u32 gro_or_buffering_if_count;
/* logging */
vlib_log_class_t log_default;
virtio_if_t *interfaces;
+ u16 msg_id_base;
} virtio_main_t;
extern virtio_main_t virtio_main;
u32 idx);
clib_error_t *virtio_vring_free_tx (vlib_main_t * vm, virtio_if_t * vif,
u32 idx);
-void virtio_vring_set_numa_node (vlib_main_t * vm, virtio_if_t * vif,
- u32 idx);
-extern void virtio_free_used_desc (vlib_main_t * vm, virtio_vring_t * vring);
-extern void virtio_free_rx_buffers (vlib_main_t * vm, virtio_vring_t * vring);
+void virtio_vring_set_rx_queues (vlib_main_t *vm, virtio_if_t *vif);
+void virtio_vring_set_tx_queues (vlib_main_t *vm, virtio_if_t *vif);
+extern void virtio_free_buffers (vlib_main_t *vm, vnet_virtio_vring_t *vring);
extern void virtio_set_net_hdr_size (virtio_if_t * vif);
-extern void virtio_show (vlib_main_t * vm, u32 * hw_if_indices, u8 show_descr,
- u32 type);
+extern void virtio_show (vlib_main_t *vm, u32 *hw_if_indices, u8 show_descr,
+ virtio_if_type_t type);
+extern void virtio_set_packet_coalesce (virtio_if_t * vif);
+clib_error_t *virtio_set_packet_buffering (virtio_if_t * vif, u16 size);
extern void virtio_pci_legacy_notify_queue (vlib_main_t * vm,
- virtio_if_t * vif, u16 queue_id);
+ virtio_if_t * vif, u16 queue_id,
+ u16 queue_notify_offset);
+extern void virtio_pci_modern_notify_queue (vlib_main_t * vm,
+ virtio_if_t * vif, u16 queue_id,
+ u16 queue_notify_offset);
+extern void virtio_pre_input_node_enable (vlib_main_t *vm, virtio_if_t *vif);
+extern void virtio_pre_input_node_disable (vlib_main_t *vm, virtio_if_t *vif);
+
format_function_t format_virtio_device_name;
format_function_t format_virtio_log_name;
static_always_inline void
-virtio_kick (vlib_main_t * vm, virtio_vring_t * vring, virtio_if_t * vif)
+virtio_kick (vlib_main_t *vm, vnet_virtio_vring_t *vring, virtio_if_t *vif)
{
if (vif->type == VIRTIO_IF_TYPE_PCI)
- virtio_pci_legacy_notify_queue (vm, vif, vring->queue_id);
+ {
+ if (vif->is_modern)
+ virtio_pci_modern_notify_queue (vm, vif, vring->queue_id,
+ vring->queue_notify_offset);
+ else
+ virtio_pci_legacy_notify_queue (vm, vif, vring->queue_id,
+ vring->queue_notify_offset);
+ }
else
{
u64 x = 1;
}
}
+static_always_inline u8
+virtio_txq_is_scheduled (vnet_virtio_vring_t *vring)
+{
+ if (vring)
+ return (vring->flags & VRING_TX_SCHEDULED);
+ return 1;
+}
+
+static_always_inline void
+virtio_txq_set_scheduled (vnet_virtio_vring_t *vring)
+{
+ if (vring)
+ vring->flags |= VRING_TX_SCHEDULED;
+}
+
+static_always_inline void
+virtio_txq_clear_scheduled (vnet_virtio_vring_t *vring)
+{
+ if (vring)
+ vring->flags &= ~VRING_TX_SCHEDULED;
+}
+
+static_always_inline void
+vnet_virtio_vring_init (vnet_virtio_vring_t *vring, u16 queue_size, void *p,
+ u32 align)
+{
+ vring->queue_size = queue_size;
+ vring->desc = p;
+ vring->avail =
+ (vnet_virtio_vring_avail_t *) ((char *) p +
+ queue_size *
+ sizeof (vnet_virtio_vring_desc_t));
+ vring->used =
+ (vnet_virtio_vring_used_t
+ *) ((char *) p + ((sizeof (vnet_virtio_vring_desc_t) * queue_size +
+ sizeof (u16) * (3 + queue_size) + align - 1) &
+ ~(align - 1)));
+ vring->avail->flags = VIRTIO_RING_FLAG_MASK_INT;
+}
+
+static_always_inline u16
+vnet_virtio_vring_size (u16 queue_size, u32 align)
+{
+ return ((sizeof (vnet_virtio_vring_desc_t) * queue_size +
+ sizeof (u16) * (3 + queue_size) + align - 1) &
+ ~(align - 1)) +
+ sizeof (u16) * 3 +
+ sizeof (vnet_virtio_vring_used_elem_t) * queue_size;
+}
#define virtio_log_debug(vif, f, ...) \
{ \
Code Review / vpp.git / blobdiff