X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fplugins%2Fvmxnet3%2Fvmxnet3.h;h=75107689443f51e420c5e3fa26c3b954317fef85;hb=a57a7005d;hp=13799407d1e059f723d0f6994d1706b416eacfc5;hpb=df7f8e8cffcc43531f7daeda44d436b60e538141;p=vpp.git diff --git a/src/plugins/vmxnet3/vmxnet3.h b/src/plugins/vmxnet3/vmxnet3.h index 13799407d1e..75107689443 100644 --- a/src/plugins/vmxnet3/vmxnet3.h +++ b/src/plugins/vmxnet3/vmxnet3.h @@ -18,6 +18,7 @@ #define foreach_vmxnet3_tx_func_error \ _(ERROR_PACKETS, "error packets") \ + _(LINK_DOWN, "link down") \ _(NO_FREE_SLOTS, "no free tx slots") typedef enum @@ -42,6 +43,56 @@ enum #undef _ }; +#define foreach_vmxnet3_show_entry \ + _(RX_COMP, "rx comp") \ + _(RX_DESC0, "rx desc 0") \ + _(RX_DESC1, "rx desc 1") \ + _(TX_COMP, "tx comp") \ + _(TX_DESC, "tx desc") + +enum +{ +#define _(a, b) VMXNET3_SHOW_##a, + foreach_vmxnet3_show_entry +#undef _ +}; + +#define foreach_vmxnet3_feature_flags \ + _(0, RXCSUM, "rx checksum") \ + _(1, RSS, "RSS") \ + _(2, RXVLAN, "rx VLAN") \ + _(3, LRO, "LRO") + +enum +{ +#define _(a, b, c) VMXNET3_F_##b = (1 << a), + foreach_vmxnet3_feature_flags +#undef _ +}; + +#define foreach_vmxnet3_rss_hash_type \ + _(0, IPV4, "ipv4") \ + _(1, TCP_IPV4, "tcp ipv4") \ + _(2, IPV6, "ipv6") \ + _(3, TCP_IPV6, "tcp ipv6") + +enum +{ +#define _(a, b, c) VMXNET3_RSS_HASH_TYPE_##b = (1 << a), + foreach_vmxnet3_rss_hash_type +#undef _ +}; + +#define VMXNET3_RSS_HASH_FUNC_TOEPLITZ 1 +#define VMXNET3_RSS_MAX_KEY_SZ 40 +#define VMXNET3_RSS_MAX_IND_TABLE_SZ 128 + +#define VMXNET3_TXQ_MAX 8 +#define VMXNET3_RXQ_MAX 16 +#define VMXNET3_TX_START(vd) ((vd)->queues) +#define VMXNET3_RX_START(vd) \ + ((vd)->queues + (vd)->num_tx_queues * sizeof (vmxnet3_tx_queue)) + /* BAR 0 */ #define VMXNET3_REG_IMR 0x0000 /* Interrupt Mask Register */ #define VMXNET3_REG_TXPROD 0x0600 /* Tx Producer Index */ @@ -66,12 +117,41 @@ enum #define VMXNET3_RXF_BTYPE (1 << 14) /* rx body buffer type */ #define VMXNET3_RXF_GEN (1 << 31) /* rx generation */ + +#define VMXNET3_RXCF_CKSUM_MASK (0xFFFF) /* rx checksum mask */ +#define VMXNET3_RXCF_TUC (1 << 16) /* rx udp/tcp checksum correct */ +#define VMXNET3_RXCF_UDP (1 << 17) /* rx udp packet */ +#define VMXNET3_RXCF_TCP (1 << 18) /* rx tcp packet */ +#define VMXNET3_RXCF_IPC (1 << 19) /* rx ip checksum correct */ +#define VMXNET3_RXCF_IP6 (1 << 20) /* rx ip6 packet */ +#define VMXNET3_RXCF_IP4 (1 << 21) /* rx ip4 packet */ +#define VMXNET3_RXCF_CT (0x7F << 24) /* rx completion type 24-30, 7 bits */ #define VMXNET3_RXCF_GEN (1 << 31) /* rx completion generation */ + #define VMXNET3_RXC_INDEX (0xFFF) /* rx completion index mask */ +#define foreach_vmxnet3_offload \ + _(0, NONE, "none") \ + _(2, CSUM, "checksum") \ + _(3, TSO, "tso") + +enum +{ +#define _(a, b, c) VMXNET3_OM_##b = (a), + foreach_vmxnet3_offload +#undef _ +}; + +/* tx desc flag 0 */ #define VMXNET3_TXF_GEN (1 << 14) /* tx generation */ + +/* tx desc flag 1 */ +#define VMXNET3_TXF_OM(x) ((x) << 10) /* tx offload mode */ +#define VMXNET3_TXF_MSSCOF(x) ((x) << 18) /* tx MSS checksum offset, flags */ #define VMXNET3_TXF_EOP (1 << 12) /* tx end of packet */ #define VMXNET3_TXF_CQ (1 << 13) /* tx completion request */ + +/* tx completion flag */ #define VMXNET3_TXCF_GEN (1 << 31) /* tx completion generation */ #define VMXNET3_TXC_INDEX (0xFFF) /* tx completion index mask */ @@ -94,10 +174,17 @@ enum #define VMXNET3_GOS_TYPE_LINUX (1 << 2) #define VMXNET3_RXCL_LEN_MASK (0x3FFF) // 14 bits #define VMXNET3_RXCL_ERROR (1 << 14) -#define VMXNET3_RXCI_EOP (1 << 14) -#define VMXNET3_RXCI_SOP (1 << 15) -#define foreach_vmxnet3_device_flags \ +#define VMXNET3_RXCI_EOP (1 << 14) /* end of packet */ +#define VMXNET3_RXCI_SOP (1 << 15) /* start of packet */ +#define VMXNET3_RXCI_CNC (1 << 30) /* Checksum not calculated */ + +#define VMXNET3_RXCOMP_TYPE (3 << 24) /* RX completion descriptor */ +#define VMXNET3_RXCOMP_TYPE_LRO (4 << 24) /* RX completion descriptor for LRO */ + +#define VMXNET3_RXECF_MSS_MASK (0xFFFF) // 16 bits + +#define foreach_vmxnet3_device_flags \ _(0, INITIALIZED, "initialized") \ _(1, ERROR, "error") \ _(2, ADMIN_UP, "admin-up") \ @@ -149,7 +236,7 @@ enum _(7, GET_DEV_EXTRA_INFO, "get dev extra info") \ _(8, GET_CONF_INTR, "get conf intr") \ _(9, GET_ADAPTIVE_RING_INFO, "get adaptive ring info") \ - _(10, GET_TXDATA_DESC_SIZE, "gte txdata desc size") \ + _(10, GET_TXDATA_DESC_SIZE, "get txdata desc size") \ _(11, RESERVED5, "reserved5") enum @@ -181,9 +268,8 @@ typedef CLIB_PACKED (struct typedef CLIB_PACKED (struct { - u32 mode; - u16 multicast_len; - u16 pad; u64 multicast_address; u8 vlan_filter[512]; + u32 mode; u16 multicast_len; u16 pad; + u64 multicast_address; u8 vlan_filter[512]; }) vmxnet3_rx_filter_config; typedef CLIB_PACKED (struct @@ -224,9 +310,9 @@ typedef CLIB_PACKED (struct u64 data_address; u64 comp_address; u64 driver_data_address; u64 pad; u32 num_desc; - u32 num_data; - u32 num_comp; u32 driver_data_len; u8 intr_index; - u8 pad1[7]; + u32 num_data; u32 num_comp; u32 driver_data_len; + u8 intr_index; + u8 pad1; u16 data_address_size; u8 pad2[4]; }) vmxnet3_tx_queue_config; typedef CLIB_PACKED (struct @@ -256,10 +342,11 @@ typedef CLIB_PACKED (struct typedef CLIB_PACKED (struct { u64 desc_address[2]; - u64 comp_address; u64 driver_data_address; u64 pad; - u32 num_desc[2]; - u32 num_comp; u32 driver_data_len; u8 intr_index; - u8 pad1[7]; + u64 comp_address; u64 driver_data_address; + u64 data_address; u32 num_desc[2]; + u32 num_comp; + u32 driver_data_len; u8 intr_index; u8 pad1; + u16 data_address_size; u8 pad2[4]; }) vmxnet3_rx_queue_config; typedef CLIB_PACKED (struct @@ -280,11 +367,6 @@ typedef CLIB_PACKED (struct u8 pad[88]; }) vmxnet3_rx_queue; -typedef CLIB_PACKED (struct - { - vmxnet3_tx_queue tx; vmxnet3_rx_queue rx; - }) vmxnet3_queues; - /* * flags: * buffer length -- bits 0-13 @@ -339,6 +421,27 @@ typedef CLIB_PACKED (struct u32 flags; }) vmxnet3_rx_comp; +/* + * flags: + * mss -- bits 0 - 15 + * tcp/udp checksum correct-- bit 16 + * udp packet -- bit 17 + * tcp packet -- bit 18 + * ip checksum correct -- bit 19 + * ipv6 -- bit 20 + * ipv4 -- bit 21 + * ip fragment -- bit 22 + * frame crc correct -- bit 23 + * completion type -- bits 24-30 + * generation -- bit 31 + */ +typedef CLIB_PACKED (struct + { + u32 dword1; + u8 seg_cnt; u8 dup_ack_cnt; u16 ts_delta; u32 dword2; + u32 flags; + }) vmxnet3_rx_comp_ext; + /* * index: * TX desc index -- bits 0-11 @@ -379,6 +482,16 @@ typedef CLIB_PACKED (struct u32 flags[2]; }) vmxnet3_tx_desc; +typedef CLIB_PACKED (struct + { + u16 hash_type; + u16 hash_func; + u16 hash_key_sz; + u16 ind_table_sz; + u8 hash_key[VMXNET3_RSS_MAX_KEY_SZ]; + u8 ind_table[VMXNET3_RSS_MAX_IND_TABLE_SZ]; + }) vmxnet3_rss_shared; + typedef struct { CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); @@ -393,8 +506,8 @@ typedef struct typedef struct { CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); - u64 next; u32 gen; + u16 next; } vmxnet3_rx_comp_ring; typedef struct @@ -402,6 +515,8 @@ typedef struct CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); u16 size; u8 int_mode; + u8 buffer_pool_index; + u32 queue_index; vmxnet3_rx_ring rx_ring[VMXNET3_RX_RING_SIZE]; vmxnet3_rx_desc *rx_desc[VMXNET3_RX_RING_SIZE]; vmxnet3_rx_comp *rx_comp; @@ -420,14 +535,16 @@ typedef struct typedef struct { CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); - u64 next; u32 gen; + u16 next; } vmxnet3_tx_comp_ring; typedef struct { CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); u16 size; + u32 queue_index; + u32 reg_txprod; clib_spinlock_t lock; vmxnet3_tx_desc *tx_desc; @@ -436,11 +553,6 @@ typedef struct vmxnet3_tx_comp_ring tx_comp_ring; } vmxnet3_txq_t; -typedef CLIB_PACKED (struct - { - vmxnet3_queues queues; vmxnet3_shared shared; - }) vmxnet3_dma; - typedef struct { CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); @@ -450,6 +562,7 @@ typedef struct u32 dev_instance; u32 sw_if_index; u32 hw_if_index; + u32 numa_node; vlib_pci_dev_handle_t pci_dev_handle; vlib_pci_addr_t pci_addr; void *bar[2]; @@ -465,19 +578,22 @@ typedef struct u8 version; u8 mac_addr[6]; - /* error */ clib_error_t *error; - vmxnet3_dma *dma; - + vmxnet3_shared *driver_shared; + void *queues; + vmxnet3_rss_shared *rss; + u32 link_speed; + u8 gso_enable; + vmxnet3_tx_stats *tx_stats; + vmxnet3_rx_stats *rx_stats; } vmxnet3_device_t; typedef struct { vmxnet3_device_t *devices; - vlib_physmem_region_index_t physmem_region; - u32 physmem_region_alloc; u16 msg_id_base; + vlib_log_class_t log_default; } vmxnet3_main_t; extern vmxnet3_main_t vmxnet3_main; @@ -487,7 +603,11 @@ typedef struct vlib_pci_addr_t addr; u32 enable_elog; u16 rxq_size; + u16 rxq_num; u16 txq_size; + u16 txq_num; + u8 bind; + u8 enable_gso; /* return */ i32 rv; u32 sw_if_index; @@ -513,25 +633,46 @@ format_function_t format_vmxnet3_device; format_function_t format_vmxnet3_device_name; format_function_t format_vmxnet3_input_trace; +#define vmxnet3_log_debug(dev, f, ...) \ + vlib_log (VLIB_LOG_LEVEL_DEBUG, vmxnet3_main.log_default, "%U: " f, \ + format_vlib_pci_addr, &dev->pci_addr, \ + ## __VA_ARGS__) + +#define vmxnet3_log_error(dev, f, ...) \ + vlib_log (VLIB_LOG_LEVEL_ERR, vmxnet3_main.log_default, "%U: " f, \ + format_vlib_pci_addr, &dev->pci_addr, \ + ## __VA_ARGS__) + +/* no log version, called by data plane */ static_always_inline void -vmxnet3_reg_write (vmxnet3_device_t * vd, u8 bar, u32 addr, u32 val) +vmxnet3_reg_write_inline (vmxnet3_device_t * vd, u8 bar, u32 addr, u32 val) { *(volatile u32 *) ((u8 *) vd->bar[bar] + addr) = val; } +static_always_inline void +vmxnet3_reg_write (vmxnet3_device_t * vd, u8 bar, u32 addr, u32 val) +{ + vmxnet3_log_debug (vd, "reg wr bar %u addr 0x%x val 0x%x", bar, addr, val); + vmxnet3_reg_write_inline (vd, bar, addr, val); +} + static_always_inline u32 vmxnet3_reg_read (vmxnet3_device_t * vd, u8 bar, u32 addr) { - return *(volatile u32 *) (vd->bar[bar] + addr); + u32 val; + + val = *(volatile u32 *) (vd->bar[bar] + addr); + vmxnet3_log_debug (vd, "reg rd bar %u addr 0x%x val 0x%x", bar, addr, val); + + return val; } static_always_inline uword vmxnet3_dma_addr (vlib_main_t * vm, vmxnet3_device_t * vd, void *p) { - vmxnet3_main_t *vmxm = &vmxnet3_main; - return (vd->flags & VMXNET3_DEVICE_F_IOVA) ? pointer_to_uword (p) : - vlib_physmem_virtual_to_physical (vm, vmxm->physmem_region, p); + vlib_physmem_get_pa (vm, p); } static_always_inline void @@ -552,6 +693,7 @@ vmxnet3_rxq_refill_ring0 (vlib_main_t * vm, vmxnet3_device_t * vd, vmxnet3_rx_desc *rxd; u16 n_refill, n_alloc; vmxnet3_rx_ring *ring; + vmxnet3_rx_queue *rx; ring = &rxq->rx_ring[0]; n_refill = rxq->size - ring->fill; @@ -560,8 +702,9 @@ vmxnet3_rxq_refill_ring0 (vlib_main_t * vm, vmxnet3_device_t * vd, return 0; n_alloc = - vlib_buffer_alloc_to_ring (vm, ring->bufs, ring->produce, rxq->size, - n_refill); + vlib_buffer_alloc_to_ring_from_pool (vm, ring->bufs, ring->produce, + rxq->size, n_refill, + rxq->buffer_pool_index); if (PREDICT_FALSE (n_alloc != n_refill)) { if (n_alloc) @@ -572,17 +715,19 @@ vmxnet3_rxq_refill_ring0 (vlib_main_t * vm, vmxnet3_device_t * vd, while (n_alloc) { + vlib_buffer_t *b = vlib_get_buffer (vm, ring->bufs[ring->produce]); rxd = &rxq->rx_desc[0][ring->produce]; - rxd->address = - vlib_get_buffer_data_physical_address (vm, ring->bufs[ring->produce]); - rxd->flags = ring->gen | VLIB_BUFFER_DATA_SIZE; + rxd->address = vlib_buffer_get_pa (vm, b); + rxd->flags = ring->gen | vlib_buffer_get_default_data_size (vm); vmxnet3_rx_ring_advance_produce (rxq, ring); ring->fill++; n_alloc--; } - vmxnet3_reg_write (vd, 0, VMXNET3_REG_RXPROD, ring->produce); + rx = VMXNET3_RX_START (vd); + if (PREDICT_FALSE (rx->ctrl.update_prod)) + vmxnet3_reg_write_inline (vd, 0, VMXNET3_REG_RXPROD, ring->produce); return 0; } @@ -594,6 +739,7 @@ vmxnet3_rxq_refill_ring1 (vlib_main_t * vm, vmxnet3_device_t * vd, vmxnet3_rx_desc *rxd; u16 n_refill, n_alloc; vmxnet3_rx_ring *ring; + vmxnet3_rx_queue *rx; ring = &rxq->rx_ring[1]; n_refill = rxq->size - ring->fill; @@ -602,8 +748,9 @@ vmxnet3_rxq_refill_ring1 (vlib_main_t * vm, vmxnet3_device_t * vd, return 0; n_alloc = - vlib_buffer_alloc_to_ring (vm, ring->bufs, ring->produce, rxq->size, - n_refill); + vlib_buffer_alloc_to_ring_from_pool (vm, ring->bufs, ring->produce, + rxq->size, n_refill, + rxq->buffer_pool_index); if (PREDICT_FALSE (n_alloc != n_refill)) { if (n_alloc) @@ -614,17 +761,20 @@ vmxnet3_rxq_refill_ring1 (vlib_main_t * vm, vmxnet3_device_t * vd, while (n_alloc) { + vlib_buffer_t *b = vlib_get_buffer (vm, ring->bufs[ring->produce]); rxd = &rxq->rx_desc[1][ring->produce]; - rxd->address = - vlib_get_buffer_data_physical_address (vm, ring->bufs[ring->produce]); - rxd->flags = ring->gen | VLIB_BUFFER_DATA_SIZE | VMXNET3_RXF_BTYPE; + rxd->address = vlib_buffer_get_pa (vm, b); + rxd->flags = ring->gen | vlib_buffer_get_default_data_size (vm) | + VMXNET3_RXF_BTYPE; vmxnet3_rx_ring_advance_produce (rxq, ring); ring->fill++; n_alloc--; } - vmxnet3_reg_write (vd, 0, VMXNET3_REG_RXPROD2, ring->produce); + rx = VMXNET3_RX_START (vd); + if (PREDICT_FALSE (rx->ctrl.update_prod)) + vmxnet3_reg_write_inline (vd, 0, VMXNET3_REG_RXPROD2, ring->produce); return 0; }