2 * Copyright (c) 2018 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #ifndef __included_vmnet_vmnet_h__
17 #define __included_vmnet_vmnet_h__
19 #define foreach_vmxnet3_tx_func_error \
20 _(ERROR_PACKETS, "error packets") \
21 _(LINK_DOWN, "link down") \
22 _(NO_FREE_SLOTS, "no free tx slots")
26 #define _(f,s) VMXNET3_TX_ERROR_##f,
27 foreach_vmxnet3_tx_func_error
30 } vmxnet3_tx_func_error_t;
32 #define foreach_vmxnet3_rxmode_flags \
33 _(0, UCAST, "unicast") \
34 _(1, MCAST, "multicast") \
35 _(2, BCAST, "broadcast") \
36 _(3, ALL_MULTI, "all multicast") \
37 _(4, PROMISC, "promiscuous")
41 #define _(a, b, c) VMXNET3_RXMODE_##b = (1 << a),
42 foreach_vmxnet3_rxmode_flags
46 #define foreach_vmxnet3_show_entry \
47 _(RX_COMP, "rx comp") \
48 _(RX_DESC0, "rx desc 0") \
49 _(RX_DESC1, "rx desc 1") \
50 _(TX_COMP, "tx comp") \
55 #define _(a, b) VMXNET3_SHOW_##a,
56 foreach_vmxnet3_show_entry
60 #define foreach_vmxnet3_feature_flags \
61 _(0, RXCSUM, "rx checksum") \
63 _(2, RXVLAN, "rx VLAN") \
68 #define _(a, b, c) VMXNET3_F_##b = (1 << a),
69 foreach_vmxnet3_feature_flags
73 #define foreach_vmxnet3_rss_hash_type \
75 _(1, TCP_IPV4, "tcp ipv4") \
77 _(3, TCP_IPV6, "tcp ipv6")
81 #define _(a, b, c) VMXNET3_RSS_HASH_TYPE_##b = (1 << a),
82 foreach_vmxnet3_rss_hash_type
86 #define VMXNET3_RSS_HASH_FUNC_TOEPLITZ 1
87 #define VMXNET3_RSS_MAX_KEY_SZ 40
88 #define VMXNET3_RSS_MAX_IND_TABLE_SZ 128
90 #define VMXNET3_TXQ_MAX 8
91 #define VMXNET3_RXQ_MAX 16
92 #define VMXNET3_TX_START(vd) ((vd)->queues)
93 #define VMXNET3_RX_START(vd) \
94 ((vd)->queues + (vd)->num_tx_queues * sizeof (vmxnet3_tx_queue))
97 #define VMXNET3_REG_IMR 0x0000 /* Interrupt Mask Register */
98 #define VMXNET3_REG_TXPROD 0x0600 /* Tx Producer Index */
99 #define VMXNET3_REG_RXPROD 0x0800 /* Rx Producer Index for ring 1 */
100 #define VMXNET3_REG_RXPROD2 0x0A00 /* Rx Producer Index for ring 2 */
104 #define VMXNET3_REG_VRRS 0x0000 /* VMXNET3 Revision Report Selection */
105 #define VMXNET3_REG_UVRS 0x0008 /* UPT Version Report Selection */
106 #define VMXNET3_REG_DSAL 0x0010 /* Driver Shared Address Low */
107 #define VMXNET3_REG_DSAH 0x0018 /* Driver Shared Address High */
108 #define VMXNET3_REG_CMD 0x0020 /* Command */
109 #define VMXNET3_REG_MACL 0x0028 /* MAC Address Low */
110 #define VMXNET3_REG_MACH 0x0030 /* MAC Address High */
111 #define VMXNET3_REG_ICR 0x0038 /* Interrupt Cause Register */
112 #define VMXNET3_REG_ECR 0x0040 /* Event Cause Register */
114 #define VMXNET3_VLAN_LEN 4
115 #define VMXNET3_FCS_LEN 4
116 #define VMXNET3_MTU (1514 + VMXNET3_VLAN_LEN + VMXNET3_FCS_LEN)
118 #define VMXNET3_RXF_BTYPE (1 << 14) /* rx body buffer type */
119 #define VMXNET3_RXF_GEN (1 << 31) /* rx generation */
121 #define VMXNET3_RXCF_CKSUM_MASK (0xFFFF) /* rx checksum mask */
122 #define VMXNET3_RXCF_TUC (1 << 16) /* rx udp/tcp checksum correct */
123 #define VMXNET3_RXCF_UDP (1 << 17) /* rx udp packet */
124 #define VMXNET3_RXCF_TCP (1 << 18) /* rx tcp packet */
125 #define VMXNET3_RXCF_IPC (1 << 19) /* rx ip checksum correct */
126 #define VMXNET3_RXCF_IP6 (1 << 20) /* rx ip6 packet */
127 #define VMXNET3_RXCF_IP4 (1 << 21) /* rx ip4 packet */
128 #define VMXNET3_RXCF_CT (0x7F << 24) /* rx completion type 24-30, 7 bits */
129 #define VMXNET3_RXCF_GEN (1 << 31) /* rx completion generation */
131 #define VMXNET3_RXC_INDEX (0xFFF) /* rx completion index mask */
133 #define foreach_vmxnet3_offload \
135 _(2, CSUM, "checksum") \
140 #define _(a, b, c) VMXNET3_OM_##b = (a),
141 foreach_vmxnet3_offload
146 #define VMXNET3_TXF_GEN (1 << 14) /* tx generation */
149 #define VMXNET3_TXF_OM(x) ((x) << 10) /* tx offload mode */
150 #define VMXNET3_TXF_MSSCOF(x) ((x) << 18) /* tx MSS checksum offset, flags */
151 #define VMXNET3_TXF_EOP (1 << 12) /* tx end of packet */
152 #define VMXNET3_TXF_CQ (1 << 13) /* tx completion request */
154 /* tx completion flag */
155 #define VMXNET3_TXCF_GEN (1 << 31) /* tx completion generation */
156 #define VMXNET3_TXC_INDEX (0xFFF) /* tx completion index mask */
158 #define VMXNET3_RX_RING_SIZE 2
159 #define VMXNET3_INPUT_REFILL_THRESHOLD 32
160 #define VMXNET3_NUM_TX_DESC 1024
161 #define VMXNET3_NUM_TX_COMP VMXNET3_NUM_TX_DESC
162 #define VMXNET3_NUM_RX_DESC 1024
163 #define VMXNET3_NUM_RX_COMP VMXNET3_NUM_RX_DESC
165 #define VMXNET3_VERSION_MAGIC 0x69505845
166 #define VMXNET3_SHARED_MAGIC 0xbabefee1
167 #define VMXNET3_VERSION_SELECT 1
168 #define VMXNET3_UPT_VERSION_SELECT 1
169 #define VMXNET3_MAX_INTRS 25
170 #define VMXNET3_IC_DISABLE_ALL 0x1
172 #define VMXNET3_GOS_BITS_32 (1 << 0)
173 #define VMXNET3_GOS_BITS_64 (2 << 0)
174 #define VMXNET3_GOS_TYPE_LINUX (1 << 2)
175 #define VMXNET3_RXCL_LEN_MASK (0x3FFF) // 14 bits
176 #define VMXNET3_RXCL_ERROR (1 << 14)
178 #define VMXNET3_RXCI_EOP (1 << 14) /* end of packet */
179 #define VMXNET3_RXCI_SOP (1 << 15) /* start of packet */
180 #define VMXNET3_RXCI_CNC (1 << 30) /* Checksum not calculated */
182 #define VMXNET3_RXCOMP_TYPE (3 << 24) /* RX completion descriptor */
183 #define VMXNET3_RXCOMP_TYPE_LRO (4 << 24) /* RX completion descriptor for LRO */
185 #define VMXNET3_RXECF_MSS_MASK (0xFFFF) // 16 bits
187 #define foreach_vmxnet3_device_flags \
188 _(0, INITIALIZED, "initialized") \
189 _(1, ERROR, "error") \
190 _(2, ADMIN_UP, "admin-up") \
192 _(4, LINK_UP, "link-up") \
193 _(5, SHARED_TXQ_LOCK, "shared-txq-lock") \
198 #define _(a, b, c) VMXNET3_DEVICE_F_##b = (1 << a),
199 foreach_vmxnet3_device_flags
203 #define foreach_vmxnet3_set_cmds \
204 _(0, ACTIVATE_DEV, "activate device") \
205 _(1, QUIESCE_DEV, "quiesce device") \
206 _(2, RESET_DEV, "reset device") \
207 _(3, UPDATE_RX_MODE, "update rx mode") \
208 _(4, UPDATE_MAC_FILTERS, "update mac filters") \
209 _(5, UPDATE_VLAN_FILTERS, "update vlan filters") \
210 _(6, UPDATE_RSSIDT, "update rss idt") \
211 _(7, UPDATE_IML, "update iml") \
212 _(8, UPDATE_PMCFG, "update pm cfg") \
213 _(9, UPDATE_FEATURE, "update feature") \
214 _(10, STOP_EMULATION, "stop emulation") \
215 _(11, LOAD_PLUGIN, "load plugin") \
216 _(12, ACTIVATE_VF, "activate vf") \
217 _(13, RESERVED3, "reserved 3") \
218 _(14, RESERVED4, "reservced 4") \
219 _(15, REGISTER_MEMREGS, "register mem regs")
223 #define _(a, b, c) VMXNET3_CMD_##b = (a + 0xCAFE0000),
224 foreach_vmxnet3_set_cmds
228 #define foreach_vmxnet3_get_cmds \
229 _(0, GET_QUEUE_STATUS, "get queue status") \
230 _(1, GET_STATS, "get stats") \
231 _(2, GET_LINK, "get link") \
232 _(3, GET_PERM_MAC_LO, "get perm mac lo") \
233 _(4, GET_PERM_MAC_HI, "get perm mac hi") \
234 _(5, GET_DID_LO, "get did lo") \
235 _(6, GET_DID_HI, "get did hi") \
236 _(7, GET_DEV_EXTRA_INFO, "get dev extra info") \
237 _(8, GET_CONF_INTR, "get conf intr") \
238 _(9, GET_ADAPTIVE_RING_INFO, "get adaptive ring info") \
239 _(10, GET_TXDATA_DESC_SIZE, "get txdata desc size") \
240 _(11, RESERVED5, "reserved5")
244 #define _(a, b, c) VMXNET3_CMD_##b = (a + 0xF00D0000),
245 foreach_vmxnet3_get_cmds
249 typedef CLIB_PACKED (struct
251 u32 version; u32 guest_info; u32 version_support;
252 u32 upt_version_support; u64 upt_features;
253 u64 driver_data_address; u64 queue_desc_address;
254 u32 driver_data_len; u32 queue_desc_len;
256 u16 max_num_rx_sg; u8 num_tx_queues; u8 num_rx_queues;
258 }) vmxnet3_misc_config;
260 typedef CLIB_PACKED (struct
265 u8 moderation_level[VMXNET3_MAX_INTRS]; u32 control;
267 }) vmxnet3_interrupt_config;
269 typedef CLIB_PACKED (struct
271 u32 mode; u16 multicast_len; u16 pad;
272 u64 multicast_address; u8 vlan_filter[512];
273 }) vmxnet3_rx_filter_config;
275 typedef CLIB_PACKED (struct
277 u32 version; u32 length;
279 }) vmxnet3_variable_config;
281 typedef CLIB_PACKED (struct
285 vmxnet3_misc_config misc;
286 vmxnet3_interrupt_config interrupt;
287 vmxnet3_rx_filter_config rx_filter;
288 vmxnet3_variable_config rss;
289 vmxnet3_variable_config pattern;
290 vmxnet3_variable_config plugin; u32 ecr;
294 typedef CLIB_PACKED (struct
299 }) vmxnet3_queue_status;
301 typedef CLIB_PACKED (struct
303 u32 num_deferred; u32 threshold;
305 }) vmxnet3_tx_queue_control;
307 typedef CLIB_PACKED (struct
311 u64 comp_address; u64 driver_data_address; u64 pad;
313 u32 num_data; u32 num_comp; u32 driver_data_len;
315 u8 pad1; u16 data_address_size; u8 pad2[4];
316 }) vmxnet3_tx_queue_config;
318 typedef CLIB_PACKED (struct
322 u64 ucast_pkts; u64 ucast_bytes; u64 mcast_pkts;
324 u64 bcast_pkts; u64 bcast_bytes; u64 error_pkts;
328 typedef CLIB_PACKED (struct
330 vmxnet3_tx_queue_control ctrl;
331 vmxnet3_tx_queue_config cfg;
332 vmxnet3_queue_status status; vmxnet3_tx_stats stats;
336 typedef CLIB_PACKED (struct
338 u8 update_prod; u8 pad[7];
340 }) vmxnet3_rx_queue_control;
342 typedef CLIB_PACKED (struct
345 u64 comp_address; u64 driver_data_address;
346 u64 data_address; u32 num_desc[2];
348 u32 driver_data_len; u8 intr_index; u8 pad1;
349 u16 data_address_size; u8 pad2[4];
350 }) vmxnet3_rx_queue_config;
352 typedef CLIB_PACKED (struct
356 u64 ucast_pkts; u64 ucast_bytes; u64 mcast_pkts;
358 u64 bcast_pkts; u64 bcast_bytes; u64 nobuf_pkts;
362 typedef CLIB_PACKED (struct
364 vmxnet3_rx_queue_control ctrl;
365 vmxnet3_rx_queue_config cfg;
366 vmxnet3_queue_status status; vmxnet3_rx_stats stats;
372 * buffer length -- bits 0-13
373 * buffer type -- bit 14
374 * descriptor type -- bit 15
375 * reserved -- bits 16-30
376 * generation -- bit 31
378 typedef CLIB_PACKED (struct
387 * RX desc index -- bits 0-11
389 * end of packet -- bit 14
390 * start of packet -- bit 15
391 * ring ID -- bits 16-25
392 * RSS hash type -- bits 26-29
393 * checksum not calculated -- bit 30
396 * rss: RSS hash value
399 * data length -- bits 0-13
401 * tag is stripped -- bit 15
402 * tag stripped -- bits 16-31
405 * checksum -- bits 0 - 15
406 * tcp/udp checksum correct-- bit 16
407 * udp packet -- bit 17
408 * tcp packet -- bit 18
409 * ip checksum correct -- bit 19
412 * ip fragment -- bit 22
413 * frame crc correct -- bit 23
414 * completion type -- bits 24-30
415 * generation -- bit 31
417 typedef CLIB_PACKED (struct
427 * tcp/udp checksum correct-- bit 16
428 * udp packet -- bit 17
429 * tcp packet -- bit 18
430 * ip checksum correct -- bit 19
433 * ip fragment -- bit 22
434 * frame crc correct -- bit 23
435 * completion type -- bits 24-30
436 * generation -- bit 31
438 typedef CLIB_PACKED (struct
441 u8 seg_cnt; u8 dup_ack_cnt; u16 ts_delta; u32 dword2;
443 }) vmxnet3_rx_comp_ext;
447 * TX desc index -- bits 0-11
451 * reserved -- bits 0-23
452 * completion type -- bits 24-30
453 * generation -- bit 31
455 typedef CLIB_PACKED (struct
464 * length -- bits 0-13
465 * generation -- bit 14
467 * descriptor type -- bit 16
469 * MSS, checksum offset -- bits 18-31
471 * header length -- bits 0-9
472 * offload mode -- bits 10-11
473 * end of packet -- bit 12
474 * completion request -- bit 13
476 * vlan tag insertion -- bit 15
477 * tag to insert -- bits 16-31
479 typedef CLIB_PACKED (struct
485 typedef CLIB_PACKED (struct
491 u8 hash_key[VMXNET3_RSS_MAX_KEY_SZ];
492 u8 ind_table[VMXNET3_RSS_MAX_IND_TABLE_SZ];
493 }) vmxnet3_rss_shared;
497 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
508 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
511 } vmxnet3_rx_comp_ring;
515 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
518 u8 buffer_pool_index;
519 vmxnet3_rx_ring rx_ring[VMXNET3_RX_RING_SIZE];
520 vmxnet3_rx_desc *rx_desc[VMXNET3_RX_RING_SIZE];
521 vmxnet3_rx_comp *rx_comp;
522 vmxnet3_rx_comp_ring rx_comp_ring;
527 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
536 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
539 } vmxnet3_tx_comp_ring;
543 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
546 clib_spinlock_t lock;
548 vmxnet3_tx_desc *tx_desc;
549 vmxnet3_tx_comp *tx_comp;
550 vmxnet3_tx_ring tx_ring;
551 vmxnet3_tx_comp_ring tx_comp_ring;
556 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
558 u32 per_interface_next_index;
564 vlib_pci_dev_handle_t pci_dev_handle;
565 vlib_pci_addr_t pci_addr;
581 vmxnet3_shared *driver_shared;
583 vmxnet3_rss_shared *rss;
586 vmxnet3_tx_stats *tx_stats;
587 vmxnet3_rx_stats *rx_stats;
592 vmxnet3_device_t *devices;
594 vlib_log_class_t log_default;
598 extern vmxnet3_main_t vmxnet3_main;
602 vlib_pci_addr_t addr;
613 } vmxnet3_create_if_args_t;
619 vlib_buffer_t buffer;
620 } vmxnet3_input_trace_t;
622 void vmxnet3_create_if (vlib_main_t * vm, vmxnet3_create_if_args_t * args);
623 void vmxnet3_delete_if (vlib_main_t * vm, vmxnet3_device_t * ad);
625 extern clib_error_t *vmxnet3_plugin_api_hookup (vlib_main_t * vm);
626 extern vlib_node_registration_t vmxnet3_input_node;
627 extern vnet_device_class_t vmxnet3_device_class;
630 format_function_t format_vmxnet3_device;
631 format_function_t format_vmxnet3_device_name;
632 format_function_t format_vmxnet3_input_trace;
634 #define vmxnet3_log_debug(dev, f, ...) \
635 vlib_log (VLIB_LOG_LEVEL_DEBUG, vmxnet3_main.log_default, "%U: " f, \
636 format_vlib_pci_addr, &dev->pci_addr, \
639 #define vmxnet3_log_error(dev, f, ...) \
640 vlib_log (VLIB_LOG_LEVEL_ERR, vmxnet3_main.log_default, "%U: " f, \
641 format_vlib_pci_addr, &dev->pci_addr, \
644 /* no log version, called by data plane */
645 static_always_inline void
646 vmxnet3_reg_write_inline (vmxnet3_device_t * vd, u8 bar, u32 addr, u32 val)
648 *(volatile u32 *) ((u8 *) vd->bar[bar] + addr) = val;
651 static_always_inline void
652 vmxnet3_reg_write (vmxnet3_device_t * vd, u8 bar, u32 addr, u32 val)
654 vmxnet3_log_debug (vd, "reg wr bar %u addr 0x%x val 0x%x", bar, addr, val);
655 vmxnet3_reg_write_inline (vd, bar, addr, val);
658 static_always_inline u32
659 vmxnet3_reg_read (vmxnet3_device_t * vd, u8 bar, u32 addr)
663 val = *(volatile u32 *) (vd->bar[bar] + addr);
664 vmxnet3_log_debug (vd, "reg rd bar %u addr 0x%x val 0x%x", bar, addr, val);
669 static_always_inline uword
670 vmxnet3_dma_addr (vlib_main_t * vm, vmxnet3_device_t * vd, void *p)
672 return (vd->flags & VMXNET3_DEVICE_F_IOVA) ? pointer_to_uword (p) :
673 vlib_physmem_get_pa (vm, p);
676 static_always_inline void
677 vmxnet3_rx_ring_advance_produce (vmxnet3_rxq_t * rxq, vmxnet3_rx_ring * ring)
680 if (PREDICT_FALSE (ring->produce == rxq->size))
683 ring->gen ^= VMXNET3_RXF_GEN;
687 static_always_inline clib_error_t *
688 vmxnet3_rxq_refill_ring0 (vlib_main_t * vm, vmxnet3_device_t * vd,
691 vmxnet3_rx_desc *rxd;
692 u16 n_refill, n_alloc;
693 vmxnet3_rx_ring *ring;
694 vmxnet3_rx_queue *rx;
696 ring = &rxq->rx_ring[0];
697 n_refill = rxq->size - ring->fill;
699 if (PREDICT_TRUE (n_refill <= VMXNET3_INPUT_REFILL_THRESHOLD))
703 vlib_buffer_alloc_to_ring_from_pool (vm, ring->bufs, ring->produce,
705 rxq->buffer_pool_index);
706 if (PREDICT_FALSE (n_alloc != n_refill))
709 vlib_buffer_free_from_ring (vm, ring->bufs, ring->produce, rxq->size,
711 return clib_error_return (0, "buffer alloc failed");
716 vlib_buffer_t *b = vlib_get_buffer (vm, ring->bufs[ring->produce]);
717 rxd = &rxq->rx_desc[0][ring->produce];
718 rxd->address = vlib_buffer_get_pa (vm, b);
719 rxd->flags = ring->gen | vlib_buffer_get_default_data_size (vm);
721 vmxnet3_rx_ring_advance_produce (rxq, ring);
726 rx = VMXNET3_RX_START (vd);
727 if (PREDICT_FALSE (rx->ctrl.update_prod))
728 vmxnet3_reg_write_inline (vd, 0, VMXNET3_REG_RXPROD, ring->produce);
733 static_always_inline clib_error_t *
734 vmxnet3_rxq_refill_ring1 (vlib_main_t * vm, vmxnet3_device_t * vd,
737 vmxnet3_rx_desc *rxd;
738 u16 n_refill, n_alloc;
739 vmxnet3_rx_ring *ring;
740 vmxnet3_rx_queue *rx;
742 ring = &rxq->rx_ring[1];
743 n_refill = rxq->size - ring->fill;
745 if (PREDICT_TRUE (n_refill <= VMXNET3_INPUT_REFILL_THRESHOLD))
749 vlib_buffer_alloc_to_ring_from_pool (vm, ring->bufs, ring->produce,
751 rxq->buffer_pool_index);
752 if (PREDICT_FALSE (n_alloc != n_refill))
755 vlib_buffer_free_from_ring (vm, ring->bufs, ring->produce, rxq->size,
757 return clib_error_return (0, "buffer alloc failed");
762 vlib_buffer_t *b = vlib_get_buffer (vm, ring->bufs[ring->produce]);
763 rxd = &rxq->rx_desc[1][ring->produce];
764 rxd->address = vlib_buffer_get_pa (vm, b);
765 rxd->flags = ring->gen | vlib_buffer_get_default_data_size (vm) |
768 vmxnet3_rx_ring_advance_produce (rxq, ring);
773 rx = VMXNET3_RX_START (vd);
774 if (PREDICT_FALSE (rx->ctrl.update_prod))
775 vmxnet3_reg_write_inline (vd, 0, VMXNET3_REG_RXPROD2, ring->produce);
780 #endif /* __included_vmnet_vmnet_h__ */
782 * fd.io coding-style-patch-verification: ON
785 * eval: (c-set-style "gnu")