#undef _
};
+#define foreach_vmxnet3_feature_flags \
+ _(0, RXCSUM, "rx checksum") \
+ _(1, RSS, "RSS") \
+ _(2, RXVLAN, "rx VLAN") \
+ _(3, LRO, "LRO")
+
+enum
+{
+#define _(a, b, c) VMXNET3_F_##b = (1 << a),
+ foreach_vmxnet3_feature_flags
+#undef _
+};
+
+#define foreach_vmxnet3_rss_hash_type \
+ _(0, IPV4, "ipv4") \
+ _(1, TCP_IPV4, "tcp ipv4") \
+ _(2, IPV6, "ipv6") \
+ _(3, TCP_IPV6, "tcp ipv6")
+
+enum
+{
+#define _(a, b, c) VMXNET3_RSS_HASH_TYPE_##b = (1 << a),
+ foreach_vmxnet3_rss_hash_type
+#undef _
+};
+
+#define VMXNET3_RSS_HASH_FUNC_TOEPLITZ 1
+#define VMXNET3_RSS_MAX_KEY_SZ 40
+#define VMXNET3_RSS_MAX_IND_TABLE_SZ 128
+
+#define VMXNET3_TXQ_MAX 8
+#define VMXNET3_RXQ_MAX 16
+#define VMXNET3_TX_START(vd) ((vd)->queues)
+#define VMXNET3_RX_START(vd) \
+ ((vd)->queues + (vd)->num_tx_queues * sizeof (vmxnet3_tx_queue))
+
/* BAR 0 */
#define VMXNET3_REG_IMR 0x0000 /* Interrupt Mask Register */
#define VMXNET3_REG_TXPROD 0x0600 /* Tx Producer Index */
#define VMXNET3_RXF_BTYPE (1 << 14) /* rx body buffer type */
#define VMXNET3_RXF_GEN (1 << 31) /* rx generation */
+
+#define VMXNET3_RXCF_CKSUM_MASK (0xFFFF) /* rx checksum mask */
+#define VMXNET3_RXCF_TUC (1 << 16) /* rx udp/tcp checksum correct */
+#define VMXNET3_RXCF_UDP (1 << 17) /* rx udp packet */
+#define VMXNET3_RXCF_TCP (1 << 18) /* rx tcp packet */
+#define VMXNET3_RXCF_IPC (1 << 19) /* rx ip checksum correct */
#define VMXNET3_RXCF_IP6 (1 << 20) /* rx ip6 packet */
#define VMXNET3_RXCF_IP4 (1 << 21) /* rx ip4 packet */
+#define VMXNET3_RXCF_CT (0x7F << 24) /* rx completion type 24-30, 7 bits */
#define VMXNET3_RXCF_GEN (1 << 31) /* rx completion generation */
+
#define VMXNET3_RXC_INDEX (0xFFF) /* rx completion index mask */
+#define foreach_vmxnet3_offload \
+ _(0, NONE, "none") \
+ _(2, CSUM, "checksum") \
+ _(3, TSO, "tso")
+
+enum
+{
+#define _(a, b, c) VMXNET3_OM_##b = (a),
+ foreach_vmxnet3_offload
+#undef _
+};
+
+/* tx desc flag 0 */
#define VMXNET3_TXF_GEN (1 << 14) /* tx generation */
+
+/* tx desc flag 1 */
+#define VMXNET3_TXF_OM(x) ((x) << 10) /* tx offload mode */
+#define VMXNET3_TXF_MSSCOF(x) ((x) << 18) /* tx MSS checksum offset, flags */
#define VMXNET3_TXF_EOP (1 << 12) /* tx end of packet */
#define VMXNET3_TXF_CQ (1 << 13) /* tx completion request */
+
+/* tx completion flag */
#define VMXNET3_TXCF_GEN (1 << 31) /* tx completion generation */
#define VMXNET3_TXC_INDEX (0xFFF) /* tx completion index mask */
#define VMXNET3_GOS_TYPE_LINUX (1 << 2)
#define VMXNET3_RXCL_LEN_MASK (0x3FFF) // 14 bits
#define VMXNET3_RXCL_ERROR (1 << 14)
-#define VMXNET3_RXCI_EOP (1 << 14)
-#define VMXNET3_RXCI_SOP (1 << 15)
-#define foreach_vmxnet3_device_flags \
+#define VMXNET3_RXCI_EOP (1 << 14) /* end of packet */
+#define VMXNET3_RXCI_SOP (1 << 15) /* start of packet */
+#define VMXNET3_RXCI_CNC (1 << 30) /* Checksum not calculated */
+
+#define VMXNET3_RXCOMP_TYPE (3 << 24) /* RX completion descriptor */
+#define VMXNET3_RXCOMP_TYPE_LRO (4 << 24) /* RX completion descriptor for LRO */
+
+#define VMXNET3_RXECF_MSS_MASK (0xFFFF) // 16 bits
+
+#define foreach_vmxnet3_device_flags \
_(0, INITIALIZED, "initialized") \
_(1, ERROR, "error") \
_(2, ADMIN_UP, "admin-up") \
_(7, GET_DEV_EXTRA_INFO, "get dev extra info") \
_(8, GET_CONF_INTR, "get conf intr") \
_(9, GET_ADAPTIVE_RING_INFO, "get adaptive ring info") \
- _(10, GET_TXDATA_DESC_SIZE, "gte txdata desc size") \
+ _(10, GET_TXDATA_DESC_SIZE, "get txdata desc size") \
_(11, RESERVED5, "reserved5")
enum
typedef CLIB_PACKED (struct
{
- u32 mode;
- u16 multicast_len;
- u16 pad; u64 multicast_address; u8 vlan_filter[512];
+ u32 mode; u16 multicast_len; u16 pad;
+ u64 multicast_address; u8 vlan_filter[512];
}) vmxnet3_rx_filter_config;
typedef CLIB_PACKED (struct
u64 data_address;
u64 comp_address; u64 driver_data_address; u64 pad;
u32 num_desc;
- u32 num_data;
- u32 num_comp; u32 driver_data_len; u8 intr_index;
- u8 pad1[7];
+ u32 num_data; u32 num_comp; u32 driver_data_len;
+ u8 intr_index;
+ u8 pad1; u16 data_address_size; u8 pad2[4];
}) vmxnet3_tx_queue_config;
typedef CLIB_PACKED (struct
typedef CLIB_PACKED (struct
{
u64 desc_address[2];
- u64 comp_address; u64 driver_data_address; u64 pad;
- u32 num_desc[2];
- u32 num_comp; u32 driver_data_len; u8 intr_index;
- u8 pad1[7];
+ u64 comp_address; u64 driver_data_address;
+ u64 data_address; u32 num_desc[2];
+ u32 num_comp;
+ u32 driver_data_len; u8 intr_index; u8 pad1;
+ u16 data_address_size; u8 pad2[4];
}) vmxnet3_rx_queue_config;
typedef CLIB_PACKED (struct
u8 pad[88];
}) vmxnet3_rx_queue;
-typedef CLIB_PACKED (struct
- {
- vmxnet3_tx_queue tx; vmxnet3_rx_queue rx;
- }) vmxnet3_queues;
-
/*
* flags:
* buffer length -- bits 0-13
u32 flags;
}) vmxnet3_rx_comp;
+/*
+ * flags:
+ * mss -- bits 0 - 15
+ * tcp/udp checksum correct-- bit 16
+ * udp packet -- bit 17
+ * tcp packet -- bit 18
+ * ip checksum correct -- bit 19
+ * ipv6 -- bit 20
+ * ipv4 -- bit 21
+ * ip fragment -- bit 22
+ * frame crc correct -- bit 23
+ * completion type -- bits 24-30
+ * generation -- bit 31
+ */
+typedef CLIB_PACKED (struct
+ {
+ u32 dword1;
+ u8 seg_cnt; u8 dup_ack_cnt; u16 ts_delta; u32 dword2;
+ u32 flags;
+ }) vmxnet3_rx_comp_ext;
+
/*
* index:
* TX desc index -- bits 0-11
u32 flags[2];
}) vmxnet3_tx_desc;
+typedef CLIB_PACKED (struct
+ {
+ u16 hash_type;
+ u16 hash_func;
+ u16 hash_key_sz;
+ u16 ind_table_sz;
+ u8 hash_key[VMXNET3_RSS_MAX_KEY_SZ];
+ u8 ind_table[VMXNET3_RSS_MAX_IND_TABLE_SZ];
+ }) vmxnet3_rss_shared;
+
typedef struct
{
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
u16 size;
u8 int_mode;
+ u8 buffer_pool_index;
+ u32 queue_index;
vmxnet3_rx_ring rx_ring[VMXNET3_RX_RING_SIZE];
vmxnet3_rx_desc *rx_desc[VMXNET3_RX_RING_SIZE];
vmxnet3_rx_comp *rx_comp;
{
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
u16 size;
+ u32 queue_index;
+ u32 reg_txprod;
clib_spinlock_t lock;
vmxnet3_tx_desc *tx_desc;
vmxnet3_tx_comp_ring tx_comp_ring;
} vmxnet3_txq_t;
-typedef CLIB_PACKED (struct
- {
- vmxnet3_queues queues; vmxnet3_shared shared;
- }) vmxnet3_dma;
-
typedef struct
{
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
u32 dev_instance;
u32 sw_if_index;
u32 hw_if_index;
+ u32 numa_node;
vlib_pci_dev_handle_t pci_dev_handle;
vlib_pci_addr_t pci_addr;
void *bar[2];
u8 version;
u8 mac_addr[6];
- /* error */
clib_error_t *error;
- vmxnet3_dma *dma;
-
+ vmxnet3_shared *driver_shared;
+ void *queues;
+ vmxnet3_rss_shared *rss;
u32 link_speed;
+ u8 gso_enable;
+ vmxnet3_tx_stats *tx_stats;
+ vmxnet3_rx_stats *rx_stats;
} vmxnet3_device_t;
typedef struct
{
vmxnet3_device_t *devices;
- vlib_physmem_region_index_t physmem_region;
- u32 physmem_region_alloc;
u16 msg_id_base;
+ vlib_log_class_t log_default;
} vmxnet3_main_t;
extern vmxnet3_main_t vmxnet3_main;
vlib_pci_addr_t addr;
u32 enable_elog;
u16 rxq_size;
+ u16 rxq_num;
u16 txq_size;
+ u16 txq_num;
+ u8 bind;
+ u8 enable_gso;
/* return */
i32 rv;
u32 sw_if_index;
format_function_t format_vmxnet3_device_name;
format_function_t format_vmxnet3_input_trace;
+#define vmxnet3_log_debug(dev, f, ...) \
+ vlib_log (VLIB_LOG_LEVEL_DEBUG, vmxnet3_main.log_default, "%U: " f, \
+ format_vlib_pci_addr, &dev->pci_addr, \
+ ## __VA_ARGS__)
+
+#define vmxnet3_log_error(dev, f, ...) \
+ vlib_log (VLIB_LOG_LEVEL_ERR, vmxnet3_main.log_default, "%U: " f, \
+ format_vlib_pci_addr, &dev->pci_addr, \
+ ## __VA_ARGS__)
+
+/* no log version, called by data plane */
static_always_inline void
-vmxnet3_reg_write (vmxnet3_device_t * vd, u8 bar, u32 addr, u32 val)
+vmxnet3_reg_write_inline (vmxnet3_device_t * vd, u8 bar, u32 addr, u32 val)
{
*(volatile u32 *) ((u8 *) vd->bar[bar] + addr) = val;
}
+static_always_inline void
+vmxnet3_reg_write (vmxnet3_device_t * vd, u8 bar, u32 addr, u32 val)
+{
+ vmxnet3_log_debug (vd, "reg wr bar %u addr 0x%x val 0x%x", bar, addr, val);
+ vmxnet3_reg_write_inline (vd, bar, addr, val);
+}
+
static_always_inline u32
vmxnet3_reg_read (vmxnet3_device_t * vd, u8 bar, u32 addr)
{
- return *(volatile u32 *) (vd->bar[bar] + addr);
+ u32 val;
+
+ val = *(volatile u32 *) (vd->bar[bar] + addr);
+ vmxnet3_log_debug (vd, "reg rd bar %u addr 0x%x val 0x%x", bar, addr, val);
+
+ return val;
}
static_always_inline uword
vmxnet3_dma_addr (vlib_main_t * vm, vmxnet3_device_t * vd, void *p)
{
- vmxnet3_main_t *vmxm = &vmxnet3_main;
-
return (vd->flags & VMXNET3_DEVICE_F_IOVA) ? pointer_to_uword (p) :
- vlib_physmem_virtual_to_physical (vm, vmxm->physmem_region, p);
+ vlib_physmem_get_pa (vm, p);
}
static_always_inline void
vmxnet3_rx_desc *rxd;
u16 n_refill, n_alloc;
vmxnet3_rx_ring *ring;
+ vmxnet3_rx_queue *rx;
ring = &rxq->rx_ring[0];
n_refill = rxq->size - ring->fill;
return 0;
n_alloc =
- vlib_buffer_alloc_to_ring (vm, ring->bufs, ring->produce, rxq->size,
- n_refill);
+ vlib_buffer_alloc_to_ring_from_pool (vm, ring->bufs, ring->produce,
+ rxq->size, n_refill,
+ rxq->buffer_pool_index);
if (PREDICT_FALSE (n_alloc != n_refill))
{
if (n_alloc)
vlib_buffer_t *b = vlib_get_buffer (vm, ring->bufs[ring->produce]);
rxd = &rxq->rx_desc[0][ring->produce];
rxd->address = vlib_buffer_get_pa (vm, b);
- rxd->flags = ring->gen | VLIB_BUFFER_DATA_SIZE;
+ rxd->flags = ring->gen | vlib_buffer_get_default_data_size (vm);
vmxnet3_rx_ring_advance_produce (rxq, ring);
ring->fill++;
n_alloc--;
}
- vmxnet3_reg_write (vd, 0, VMXNET3_REG_RXPROD, ring->produce);
+ rx = VMXNET3_RX_START (vd);
+ if (PREDICT_FALSE (rx->ctrl.update_prod))
+ vmxnet3_reg_write_inline (vd, 0, VMXNET3_REG_RXPROD, ring->produce);
return 0;
}
vmxnet3_rx_desc *rxd;
u16 n_refill, n_alloc;
vmxnet3_rx_ring *ring;
+ vmxnet3_rx_queue *rx;
ring = &rxq->rx_ring[1];
n_refill = rxq->size - ring->fill;
return 0;
n_alloc =
- vlib_buffer_alloc_to_ring (vm, ring->bufs, ring->produce, rxq->size,
- n_refill);
+ vlib_buffer_alloc_to_ring_from_pool (vm, ring->bufs, ring->produce,
+ rxq->size, n_refill,
+ rxq->buffer_pool_index);
if (PREDICT_FALSE (n_alloc != n_refill))
{
if (n_alloc)
vlib_buffer_t *b = vlib_get_buffer (vm, ring->bufs[ring->produce]);
rxd = &rxq->rx_desc[1][ring->produce];
rxd->address = vlib_buffer_get_pa (vm, b);
- rxd->flags = ring->gen | VLIB_BUFFER_DATA_SIZE | VMXNET3_RXF_BTYPE;
+ rxd->flags = ring->gen | vlib_buffer_get_default_data_size (vm) |
+ VMXNET3_RXF_BTYPE;
vmxnet3_rx_ring_advance_produce (rxq, ring);
ring->fill++;
n_alloc--;
}
- vmxnet3_reg_write (vd, 0, VMXNET3_REG_RXPROD2, ring->produce);
+ rx = VMXNET3_RX_START (vd);
+ if (PREDICT_FALSE (rx->ctrl.update_prod))
+ vmxnet3_reg_write_inline (vd, 0, VMXNET3_REG_RXPROD2, ring->produce);
return 0;
}
Code Review / vpp.git / blobdiff