2 *------------------------------------------------------------------
3 * Copyright (c) 2023 Intel and/or its affiliates.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 *------------------------------------------------------------------
21 #include <vlib/vlib.h>
22 #include <vppinfra/ring.h>
23 #include <vlib/unix/unix.h>
24 #include <vlib/pci/pci.h>
25 #include <vnet/ethernet/ethernet.h>
26 #include <vnet/interface/rx_queue_funcs.h>
27 #include <vnet/interface/tx_queue_funcs.h>
29 #include <vppinfra/types.h>
30 #include <vppinfra/error_bootstrap.h>
31 #include <vppinfra/lock.h>
34 #include <vlib/pci/pci.h>
36 #include <vnet/interface.h>
38 #include <vnet/devices/devices.h>
39 #include <vnet/flow/flow.h>
41 #include <idpf/virtchnl2.h>
42 #include <sys/queue.h>
44 #define BIT(a) (1UL << (a))
49 #define MAKEMASK(m, s) ((m) << (s))
52 #define PF_QRX_BASE 0x00000000
53 #define PF_QRX_TAIL(_QRX) (PF_QRX_BASE + (((_QRX) *0x1000)))
54 #define PF_QRX_BUFFQ_BASE 0x03000000
55 #define PF_QRX_BUFFQ_TAIL(_QRX) (PF_QRX_BUFFQ_BASE + (((_QRX) *0x1000)))
58 #define PF_QTX_BASE 0x05000000
59 #define PF_QTX_COMM_DBELL(_DBQM) (PF_QTX_BASE + ((_DBQM) *0x1000))
61 /* Control(PF Mailbox) Queue */
62 #define PF_FW_BASE 0x08400000
64 #define PF_FW_ARQBAL (PF_FW_BASE)
65 #define PF_FW_ARQBAH (PF_FW_BASE + 0x4)
66 #define PF_FW_ARQLEN (PF_FW_BASE + 0x8)
67 #define PF_FW_ARQLEN_ARQLEN_S 0
68 #define PF_FW_ARQLEN_ARQLEN_M MAKEMASK (0x1FFF, PF_FW_ARQLEN_ARQLEN_S)
69 #define PF_FW_ARQLEN_ARQVFE_S 28
70 #define PF_FW_ARQLEN_ARQVFE_M BIT (PF_FW_ARQLEN_ARQVFE_S)
71 #define PF_FW_ARQLEN_ARQOVFL_S 29
72 #define PF_FW_ARQLEN_ARQOVFL_M BIT (PF_FW_ARQLEN_ARQOVFL_S)
73 #define PF_FW_ARQLEN_ARQCRIT_S 30
74 #define PF_FW_ARQLEN_ARQCRIT_M BIT (PF_FW_ARQLEN_ARQCRIT_S)
75 #define PF_FW_ARQLEN_ARQENABLE_S 31
76 #define PF_FW_ARQLEN_ARQENABLE_M BIT (PF_FW_ARQLEN_ARQENABLE_S)
77 #define PF_FW_ARQH (PF_FW_BASE + 0xC)
78 #define PF_FW_ARQH_ARQH_S 0
79 #define PF_FW_ARQH_ARQH_M MAKEMASK (0x1FFF, PF_FW_ARQH_ARQH_S)
80 #define PF_FW_ARQT (PF_FW_BASE + 0x10)
82 #define PF_FW_ATQBAL (PF_FW_BASE + 0x14)
83 #define PF_FW_ATQBAH (PF_FW_BASE + 0x18)
84 #define PF_FW_ATQLEN (PF_FW_BASE + 0x1C)
85 #define PF_FW_ATQLEN_ATQLEN_S 0
86 #define PF_FW_ATQLEN_ATQLEN_M MAKEMASK (0x3FF, PF_FW_ATQLEN_ATQLEN_S)
87 #define PF_FW_ATQLEN_ATQVFE_S 28
88 #define PF_FW_ATQLEN_ATQVFE_M BIT (PF_FW_ATQLEN_ATQVFE_S)
89 #define PF_FW_ATQLEN_ATQOVFL_S 29
90 #define PF_FW_ATQLEN_ATQOVFL_M BIT (PF_FW_ATQLEN_ATQOVFL_S)
91 #define PF_FW_ATQLEN_ATQCRIT_S 30
92 #define PF_FW_ATQLEN_ATQCRIT_M BIT (PF_FW_ATQLEN_ATQCRIT_S)
93 #define PF_FW_ATQLEN_ATQENABLE_S 31
94 #define PF_FW_ATQLEN_ATQENABLE_M BIT (PF_FW_ATQLEN_ATQENABLE_S)
95 #define PF_FW_ATQH (PF_FW_BASE + 0x20)
96 #define PF_FW_ATQH_ATQH_S 0
97 #define PF_FW_ATQH_ATQH_M MAKEMASK (0x3FF, PF_FW_ATQH_ATQH_S)
98 #define PF_FW_ATQT (PF_FW_BASE + 0x24)
101 #define PF_GLINT_BASE 0x08900000
102 #define PF_GLINT_DYN_CTL_ITR_INDX_S 3
103 #define PF_GLINT_DYN_CTL_ITR_INDX_M MAKEMASK (0x3, PF_GLINT_DYN_CTL_ITR_INDX_S)
104 #define PF_GLINT_DYN_CTL_INTERVAL_S 5
105 #define PF_GLINT_DYN_CTL_INTERVAL_M BIT (PF_GLINT_DYN_CTL_INTERVAL_S)
106 #define PF_GLINT_DYN_CTL_WB_ON_ITR_S 30
107 #define PF_GLINT_DYN_CTL_WB_ON_ITR_M BIT (PF_GLINT_DYN_CTL_WB_ON_ITR_S)
109 /* Generic registers */
110 #define PFGEN_RSTAT 0x08407008 /* PFR Status */
111 #define PFGEN_RSTAT_PFR_STATE_S 0
112 #define PFGEN_RSTAT_PFR_STATE_M MAKEMASK (0x3, PFGEN_RSTAT_PFR_STATE_S)
113 #define PFGEN_CTRL 0x0840700C
114 #define PFGEN_CTRL_PFSWR BIT (0)
116 #define IDPF_CTLQ_ID -1
117 #define IDPF_CTLQ_LEN 64
118 #define IDPF_DFLT_MBX_BUF_SIZE 4096
120 #define IDPF_MAX_NUM_QUEUES 256
121 #define IDPF_MIN_BUF_SIZE 1024
122 #define IDPF_MAX_FRAME_SIZE 9728
123 #define IDPF_MAX_PKT_TYPE 1024
124 #define IDPF_QUEUE_SZ_MAX 4096
125 #define IDPF_QUEUE_SZ_MIN 64
127 #define IDPF_RESET_SUSPEND_TIME 20e-3
128 #define IDPF_RESET_MAX_WAIT_TIME 1
130 #define IDPF_SEND_TO_PF_SUSPEND_TIME 10e-3
131 #define IDPF_SEND_TO_PF_MAX_WAIT_TIME 1
132 #define IDPF_SEND_TO_PF_MAX_TRY_TIMES 200
134 #define IDPF_RX_MAX_DESC_IN_CHAIN 5
136 #define IDPF_MAX_VPORT_NUM 8
137 #define IDPF_DFLT_Q_VEC_NUM 1
138 #define IDPF_DFLT_INTERVAL 16
140 #define IDPF_DEFAULT_RXQ_NUM 16
141 #define IDPF_DEFAULT_TXQ_NUM 16
143 #define IDPF_ETH_ALEN 6
145 #define IDPF_INVALID_VPORT_IDX 0xffff
146 #define IDPF_TXQ_PER_GRP 1
147 #define IDPF_TX_COMPLQ_PER_GRP 1
148 #define IDPF_RXQ_PER_GRP 1
149 #define IDPF_RX_BUFQ_PER_GRP 2
150 #define IDPF_RX_BUF_STRIDE 64
152 /* Maximum buffer lengths for all control queue types */
153 #define IDPF_CTLQ_MAX_RING_SIZE 1024
154 #define IDPF_CTLQ_MAX_BUF_LEN 4096
156 #define IDPF_HI_DWORD(x) ((u32) ((((x) >> 16) >> 16) & 0xFFFFFFFF))
157 #define IDPF_LO_DWORD(x) ((u32) ((x) &0xFFFFFFFF))
158 #define IDPF_HI_WORD(x) ((u16) (((x) >> 16) & 0xFFFF))
159 #define IDPF_LO_WORD(x) ((u16) ((x) &0xFFFF))
161 #define IDPF_CTLQ_DESC(R, i) (&(((idpf_ctlq_desc_t *) ((R)->desc_ring.va))[i]))
163 #define IDPF_CTLQ_DESC_UNUSED(R) \
164 (u16) ((((R)->next_to_clean > (R)->next_to_use) ? 0 : (R)->ring_size) + \
165 (R)->next_to_clean - (R)->next_to_use - 1)
167 #define IDPF_GET_PTYPE_SIZE(p) \
168 (sizeof (virtchnl2_ptype_t) + \
169 (((p)->proto_id_count ? ((p)->proto_id_count - 1) : 0) * \
170 sizeof ((p)->proto_id[0])))
172 /* log configuration */
173 extern vlib_log_class_registration_t idpf_log;
174 extern vlib_log_class_registration_t idpf_stats_log;
176 #define idpf_log_err(dev, f, ...) \
177 vlib_log (VLIB_LOG_LEVEL_ERR, idpf_log.class, "%U: " f, \
178 format_vlib_pci_addr, &dev->pci_addr, ##__VA_ARGS__)
180 #define idpf_log_warn(dev, f, ...) \
181 vlib_log (VLIB_LOG_LEVEL_WARNING, idpf_log.class, "%U: " f, \
182 format_vlib_pci_addr, &dev->pci_addr, ##__VA_ARGS__)
184 #define idpf_log_debug(dev, f, ...) \
185 vlib_log (VLIB_LOG_LEVEL_DEBUG, idpf_log.class, "%U: " f, \
186 format_vlib_pci_addr, &dev->pci_addr, ##__VA_ARGS__)
188 #define idpf_stats_log_debug(dev, f, ...) \
189 vlib_log (VLIB_LOG_LEVEL_DEBUG, idpf_stats_log.class, "%U: " f, \
190 format_vlib_pci_addr, &dev->pci_addr, ##__VA_ARGS__)
193 #ifndef LIST_HEAD_TYPE
194 #define LIST_HEAD_TYPE(list_name, type) LIST_HEAD (list_name, type)
197 #ifndef LIST_ENTRY_TYPE
198 #define LIST_ENTRY_TYPE(type) LIST_ENTRY (type)
201 #ifndef LIST_FOR_EACH_ENTRY_SAFE
202 #define LIST_FOR_EACH_ENTRY_SAFE(pos, temp, head, entry_type, list) \
203 LIST_FOREACH (pos, head, list)
206 #ifndef LIST_FOR_EACH_ENTRY
207 #define LIST_FOR_EACH_ENTRY(pos, head, entry_type, list) \
208 LIST_FOREACH (pos, head, list)
211 #define foreach_idpf_device_flags \
212 _ (0, INITIALIZED, "initialized") \
213 _ (1, ERROR, "error") \
214 _ (2, ADMIN_UP, "admin-up") \
215 _ (3, VA_DMA, "vaddr-dma") \
216 _ (4, LINK_UP, "link-up") \
217 _ (6, ELOG, "elog") \
218 _ (7, PROMISC, "promisc") \
219 _ (8, RX_INT, "rx-interrupts") \
220 _ (9, RX_FLOW_OFFLOAD, "rx-flow-offload")
224 #define _(a, b, c) IDPF_DEVICE_F_##b = (1 << a),
225 foreach_idpf_device_flags
229 #define IDPF_PTYPE_UNKNOWN 0x00000000
230 #define IDPF_PTYPE_L2_ETHER 0x00000001
231 #define IDPF_PTYPE_L2_ETHER_TIMESYNC 0x00000002
232 #define IDPF_PTYPE_L2_ETHER_ARP 0x00000003
233 #define IDPF_PTYPE_L2_ETHER_LLDP 0x00000004
234 #define IDPF_PTYPE_L2_ETHER_NSH 0x00000005
235 #define IDPF_PTYPE_L2_ETHER_VLAN 0x00000006
236 #define IDPF_PTYPE_L2_ETHER_QINQ 0x00000007
237 #define IDPF_PTYPE_L2_ETHER_PPPOE 0x00000008
238 #define IDPF_PTYPE_L2_ETHER_FCOE 0x00000009
239 #define IDPF_PTYPE_L2_ETHER_MPLS 0x0000000a
240 #define IDPF_PTYPE_L2_MASK 0x0000000f
241 #define IDPF_PTYPE_L3_IPV4 0x00000010
242 #define IDPF_PTYPE_L3_IPV4_EXT 0x00000030
243 #define IDPF_PTYPE_L3_IPV6 0x00000040
244 #define IDPF_PTYPE_L3_IPV4_EXT_UNKNOWN 0x00000090
245 #define IDPF_PTYPE_L3_IPV6_EXT 0x000000c0
246 #define IDPF_PTYPE_L3_IPV6_EXT_UNKNOWN 0x000000e0
247 #define IDPF_PTYPE_L3_MASK 0x000000f0
248 #define IDPF_PTYPE_L4_TCP 0x00000100
249 #define IDPF_PTYPE_L4_UDP 0x00000200
250 #define IDPF_PTYPE_L4_FRAG 0x00000300
251 #define IDPF_PTYPE_L4_SCTP 0x00000400
252 #define IDPF_PTYPE_L4_ICMP 0x00000500
253 #define IDPF_PTYPE_L4_NONFRAG 0x00000600
254 #define IDPF_PTYPE_L4_IGMP 0x00000700
255 #define IDPF_PTYPE_L4_MASK 0x00000f00
256 #define IDPF_PTYPE_TUNNEL_IP 0x00001000
257 #define IDPF_PTYPE_TUNNEL_GRE 0x00002000
258 #define IDPF_PTYPE_TUNNEL_VXLAN 0x00003000
259 #define IDPF_PTYPE_TUNNEL_NVGRE 0x00004000
260 #define IDPF_PTYPE_TUNNEL_GENEVE 0x00005000
261 #define IDPF_PTYPE_TUNNEL_GRENAT 0x00006000
262 #define IDPF_PTYPE_TUNNEL_GTPC 0x00007000
263 #define IDPF_PTYPE_TUNNEL_GTPU 0x00008000
264 #define IDPF_PTYPE_TUNNEL_ESP 0x00009000
265 #define IDPF_PTYPE_TUNNEL_L2TP 0x0000a000
266 #define IDPF_PTYPE_TUNNEL_VXLAN_GPE 0x0000b000
267 #define IDPF_PTYPE_TUNNEL_MPLS_IN_GRE 0x0000c000
268 #define IDPF_PTYPE_TUNNEL_MPLS_IN_UDP 0x0000d000
269 #define IDPF_PTYPE_TUNNEL_MASK 0x0000f000
270 #define IDPF_PTYPE_INNER_L2_ETHER 0x00010000
271 #define IDPF_PTYPE_INNER_L2_ETHER_VLAN 0x00020000
272 #define IDPF_PTYPE_INNER_L2_ETHER_QINQ 0x00030000
273 #define IDPF_PTYPE_INNER_L2_MASK 0x000f0000
274 #define IDPF_PTYPE_INNER_L3_IPV4 0x00100000
275 #define IDPF_PTYPE_INNER_L3_IPV4_EXT 0x00200000
276 #define IDPF_PTYPE_INNER_L3_IPV6 0x00300000
277 #define IDPF_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN 0x00400000
278 #define IDPF_PTYPE_INNER_L3_IPV6_EXT 0x00500000
279 #define IDPF_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN 0x00600000
280 #define IDPF_PTYPE_INNER_L3_MASK 0x00f00000
281 #define IDPF_PTYPE_INNER_L4_TCP 0x01000000
282 #define IDPF_PTYPE_INNER_L4_UDP 0x02000000
283 #define IDPF_PTYPE_INNER_L4_FRAG 0x03000000
284 #define IDPF_PTYPE_INNER_L4_SCTP 0x04000000
285 #define IDPF_PTYPE_INNER_L4_ICMP 0x05000000
286 #define IDPF_PTYPE_INNER_L4_NONFRAG 0x06000000
287 #define IDPF_PTYPE_INNER_L4_MASK 0x0f000000
288 #define IDPF_PTYPE_ALL_MASK 0x0fffffff
290 /* Flags sub-structure
291 * |0 |1 |2 |3 |4 |5 |6 |7 |8 |9 |10 |11 |12 |13 |14 |15 |
292 * |DD |CMP|ERR| * RSV * |FTYPE | *RSV* |RD |VFC|BUF| HOST_ID |
294 /* command flags and offsets */
295 #define IDPF_CTLQ_FLAG_DD_S 0
296 #define IDPF_CTLQ_FLAG_CMP_S 1
297 #define IDPF_CTLQ_FLAG_ERR_S 2
298 #define IDPF_CTLQ_FLAG_FTYPE_S 6
299 #define IDPF_CTLQ_FLAG_RD_S 10
300 #define IDPF_CTLQ_FLAG_VFC_S 11
301 #define IDPF_CTLQ_FLAG_BUF_S 12
302 #define IDPF_CTLQ_FLAG_HOST_ID_S 13
304 #define IDPF_CTLQ_FLAG_DD BIT (IDPF_CTLQ_FLAG_DD_S) /* 0x1 */
305 #define IDPF_CTLQ_FLAG_CMP BIT (IDPF_CTLQ_FLAG_CMP_S) /* 0x2 */
306 #define IDPF_CTLQ_FLAG_ERR BIT (IDPF_CTLQ_FLAG_ERR_S) /* 0x4 */
307 #define IDPF_CTLQ_FLAG_FTYPE_VM \
308 BIT (IDPF_CTLQ_FLAG_FTYPE_S) /* 0x40 */
309 #define IDPF_CTLQ_FLAG_FTYPE_PF BIT (IDPF_CTLQ_FLAG_FTYPE_S + 1) /* 0x80 */
310 #define IDPF_CTLQ_FLAG_RD BIT (IDPF_CTLQ_FLAG_RD_S) /* 0x400 */
311 #define IDPF_CTLQ_FLAG_VFC BIT (IDPF_CTLQ_FLAG_VFC_S) /* 0x800 */
312 #define IDPF_CTLQ_FLAG_BUF BIT (IDPF_CTLQ_FLAG_BUF_S) /* 0x1000 */
314 /* Host ID is a special field that has 3b and not a 1b flag */
315 #define IDPF_CTLQ_FLAG_HOST_ID_M MAKE_MASK (0x7000UL, IDPF_CTLQ_FLAG_HOST_ID_S)
317 #define IDPF_FLEX_TXD_QW1_DTYPE_S 0
318 #define IDPF_FLEX_TXD_QW1_DTYPE_M MAKEMASK (0x1FUL, IDPF_FLEX_TXD_QW1_DTYPE_S)
319 #define IDPF_FLEX_TXD_QW1_CMD_S 5
320 #define IDPF_FLEX_TXD_QW1_CMD_M MAKEMASK (0x7FFUL, IDPF_FLEX_TXD_QW1_CMD_S)
322 typedef struct idpf_vport idpf_vport_t;
324 typedef volatile struct
326 u64 buf_addr; /* Packet buffer address */
332 /* DTYPE = IDPF_TX_DESC_DTYPE_FLEX_DATA_(0x03) */
335 /* DTYPE = IDPF_TX_DESC_DTYPE_FLEX_TSYN_L2TAG1 (0x06) */
343 /* DTYPE=IDPF_TX_DESC_DTYPE_FLEX_L2TAG1_L2TAG2 (0x07) */
352 } idpf_flex_tx_desc_t;
362 STATIC_ASSERT_SIZEOF (idpf_tx_desc_t, 16);
364 typedef struct idpf_rxq
366 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
367 volatile u32 *qrx_tail;
370 virtchnl2_rx_desc_t *descs;
374 u8 buffer_pool_index;
377 struct idpf_rxq *bufq1;
378 struct idpf_rxq *bufq2;
381 typedef struct idpf_txq
383 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
384 volatile u32 *qtx_tail;
388 clib_spinlock_t lock;
389 idpf_tx_desc_t *descs;
394 idpf_tx_desc_t *tmp_descs;
398 struct idpf_txq *complq;
403 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
405 u32 per_interface_next_index;
408 virtchnl2_op_t pend_cmd;
413 vlib_pci_dev_handle_t pci_dev_handle;
427 vlib_pci_addr_t pci_addr;
436 /* control queue - send and receive */
437 struct idpf_ctlq_info *asq;
438 struct idpf_ctlq_info *arq;
443 u16 subsystem_device_id;
444 u16 subsystem_vendor_id;
446 /* max config queue number per vc message */
451 idpf_vport_t **vports;
453 u16 req_vports[IDPF_MAX_VPORT_NUM];
459 u32 ptype_tbl[IDPF_MAX_PKT_TYPE];
461 /* device capability */
475 u16 default_num_vports;
479 LIST_HEAD_TYPE (list_head, idpf_ctlq_info) cq_list_head;
482 /* memory allocation tracking */
490 /* Message type read in virtual channel from PF */
493 IDPF_MSG_ERR = -1, /* Meet error when accessing admin queue */
494 IDPF_MSG_NON, /* Read nothing from admin queue */
495 IDPF_MSG_SYS, /* Read system msg from admin queue */
496 IDPF_MSG_CMD, /* Read async command result */
503 u32 tx_compl_start_qid;
504 u32 rx_buf_start_qid;
507 u32 tx_qtail_spacing;
509 u32 rx_qtail_spacing;
510 u64 tx_compl_qtail_start;
511 u32 tx_compl_qtail_spacing;
512 u64 rx_buf_qtail_start;
513 u32 rx_buf_qtail_spacing;
514 } idpf_chunks_info_t;
519 u8 *in_args; /* buffer for sending */
520 u32 in_args_size; /* buffer size for sending */
521 u8 *out_buffer; /* buffer for response */
522 u32 out_size; /* buffer size for response */
529 } idpf_vport_param_t;
534 virtchnl2_create_vport_t *vport_info;
547 u8 default_mac_addr[VIRTCHNL2_ETH_LENGTH_OF_ADDRESS];
549 u16 max_pkt_len; /* Maximum packet length */
552 virtchnl2_queue_vector_t *qv_map; /* queue vector mapping */
554 virtchnl2_alloc_vectors_t *recv_vectors;
557 idpf_chunks_info_t chunks_info;
559 virtchnl2_vport_stats_t eth_stats_offset;
562 #define IDPF_RX_VECTOR_SZ VLIB_FRAME_SIZE
566 IDPF_PROCESS_REQ_ADD_DEL_ETH_ADDR = 1,
567 IDPF_PROCESS_REQ_CONFIG_PROMISC_MDDE = 2,
568 IDPF_PROCESS_REQ_PROGRAM_FLOW = 3,
569 } idpf_process_req_type_t;
573 idpf_process_req_type_t type;
575 u32 calling_process_index;
577 int is_add, is_enable;
579 /* below parameters are used for 'program flow' event */
586 } idpf_process_req_t;
590 u64 qw1s[IDPF_RX_MAX_DESC_IN_CHAIN - 1];
591 u32 buffers[IDPF_RX_MAX_DESC_IN_CHAIN - 1];
596 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
597 vlib_buffer_t *bufs[IDPF_RX_VECTOR_SZ];
598 u16 next[IDPF_RX_VECTOR_SZ];
599 u64 qw1s[IDPF_RX_VECTOR_SZ];
600 u32 flow_ids[IDPF_RX_VECTOR_SZ];
601 idpf_rx_tail_t tails[IDPF_RX_VECTOR_SZ];
602 vlib_buffer_t buffer_template;
603 } idpf_per_thread_data_t;
609 idpf_device_t **devices;
610 idpf_per_thread_data_t *per_thread_data;
613 extern idpf_main_t idpf_main;
617 vlib_pci_addr_t addr;
629 } idpf_create_if_args_t;
631 void idpf_create_if (vlib_main_t *vm, idpf_create_if_args_t *args);
633 extern vlib_node_registration_t idpf_process_node;
634 extern vnet_device_class_t idpf_device_class;
637 format_function_t format_idpf_device_name;
638 format_function_t format_idpf_device_flags;
641 clear_cmd (idpf_device_t *id)
643 /* Return value may be checked in anither thread, need to ensure the
645 CLIB_MEMORY_BARRIER ();
646 id->pend_cmd = VIRTCHNL2_OP_UNKNOWN;
647 id->cmd_retval = VIRTCHNL2_STATUS_SUCCESS;
650 static_always_inline idpf_device_t *
651 idpf_get_device (u32 dev_instance)
653 return pool_elt_at_index (idpf_main.devices, dev_instance)[0];
657 idpf_reg_write (idpf_device_t *id, u32 addr, u32 val)
659 *(volatile u32 *) ((u8 *) id->bar0 + addr) = val;
663 idpf_reg_read (idpf_device_t *id, u32 addr)
665 u32 val = *(volatile u32 *) (id->bar0 + addr);
670 idpf_reg_flush (idpf_device_t *id)
672 idpf_reg_read (id, PFGEN_RSTAT);
673 asm volatile("" ::: "memory");
682 u64 qw1s[IDPF_RX_MAX_DESC_IN_CHAIN];
683 } idpf_input_trace_t;
686 /* Linux kernel driver can't directly use these. Instead, they are mapped to
687 * linux compatible error codes which get translated in the build script.
689 #define IDPF_SUCCESS 0
690 #define IDPF_ERR_PARAM -53 /* -EBADR */
691 #define IDPF_ERR_NOT_IMPL -95 /* -EOPNOTSUPP */
692 #define IDPF_ERR_NOT_READY -16 /* -EBUSY */
693 #define IDPF_ERR_BAD_PTR -14 /* -EFAULT */
694 #define IDPF_ERR_INVAL_SIZE -90 /* -EMSGSIZE */
695 #define IDPF_ERR_DEVICE_NOT_SUPPORTED -19 /* -ENODEV */
696 #define IDPF_ERR_FW_API_VER -13 /* -EACCESS */
697 #define IDPF_ERR_NO_MEMORY -12 /* -ENOMEM */
698 #define IDPF_ERR_CFG -22 /* -EINVAL */
699 #define IDPF_ERR_OUT_OF_RANGE -34 /* -ERANGE */
700 #define IDPF_ERR_ALREADY_EXISTS -17 /* -EEXIST */
701 #define IDPF_ERR_DOES_NOT_EXIST -6 /* -ENXIO */
702 #define IDPF_ERR_IN_USE -114 /* -EALREADY */
703 #define IDPF_ERR_MAX_LIMIT -109 /* -ETOOMANYREFS */
704 #define IDPF_ERR_RESET_ONGOING -104 /* -ECONNRESET */
706 /* CRQ/CSQ specific error codes */
707 #define IDPF_ERR_CTLQ_ERROR -74 /* -EBADMSG */
708 #define IDPF_ERR_CTLQ_TIMEOUT -110 /* -ETIMEDOUT */
709 #define IDPF_ERR_CTLQ_FULL -28 /* -ENOSPC */
710 #define IDPF_ERR_CTLQ_NO_WORK -42 /* -ENOMSG */
711 #define IDPF_ERR_CTLQ_EMPTY -105 /* -ENOBUFS */
713 /* Used for queue init, response and events */
716 IDPF_CTLQ_TYPE_MAILBOX_TX = 0,
717 IDPF_CTLQ_TYPE_MAILBOX_RX = 1,
718 IDPF_CTLQ_TYPE_CONFIG_TX = 2,
719 IDPF_CTLQ_TYPE_CONFIG_RX = 3,
720 IDPF_CTLQ_TYPE_EVENT_RX = 4,
721 IDPF_CTLQ_TYPE_RDMA_TX = 5,
722 IDPF_CTLQ_TYPE_RDMA_RX = 6,
723 IDPF_CTLQ_TYPE_RDMA_COMPL = 7
728 IDPF_PROCESS_EVENT_START = 1,
729 IDPF_PROCESS_EVENT_DELETE_IF = 2,
730 IDPF_PROCESS_EVENT_AQ_INT = 3,
731 IDPF_PROCESS_EVENT_REQ = 4,
732 } idpf_process_event_t;
735 * Generic Control Queue Structures
739 /* used for queue tracking */
742 /* Below applies only to default mb (if present) */
751 /* Generic queue msg structure */
754 u8 vmvf_type; /* represents the source of the message on recv */
755 #define IDPF_VMVF_TYPE_VF 0
756 #define IDPF_VMVF_TYPE_VM 1
757 #define IDPF_VMVF_TYPE_PF 2
759 /* 3b field used only when sending a message to peer - to be used in
760 * combination with target func_id to route the message
762 #define IDPF_HOST_ID_MASK 0x7
765 u16 data_len; /* data_len = 0 when no payload is attached */
768 u16 func_id; /* when sending a message */
769 u16 status; /* when receiving a message */
782 #define IDPF_DIRECT_CTX_SIZE 16
783 #define IDPF_INDIRECT_CTX_SIZE 8
784 /* 16 bytes of context can be provided or 8 bytes of context
785 * plus the address of a DMA buffer
787 u8 direct[IDPF_DIRECT_CTX_SIZE];
790 u8 context[IDPF_INDIRECT_CTX_SIZE];
791 idpf_dma_mem_t *payload;
796 /* Generic queue info structures */
797 /* MB, CONFIG and EVENT q do not have extended info */
800 idpf_ctlq_type_t type;
801 int id; /* absolute queue offset passed as input
802 * -1 for default mailbox if present
804 u16 len; /* Queue length passed as input */
805 u16 buf_size; /* buffer size passed as input */
806 u64 base_address; /* output, HPA of the Queue start */
807 idpf_ctlq_reg_t reg; /* registers accessed by ctlqs */
810 void *ext_info; /* Specific to q type */
811 } idpf_ctlq_create_info_t;
813 /* Control Queue information */
814 typedef struct idpf_ctlq_info
816 LIST_ENTRY_TYPE (idpf_ctlq_info) cq_list;
818 idpf_ctlq_type_t cq_type;
820 clib_spinlock_t cq_lock; /* queue lock */
822 /* used for interrupt processing */
827 idpf_dma_mem_t desc_ring; /* descriptor ring memory */
831 idpf_dma_mem_t **rx_buff;
832 idpf_ctlq_msg_t **tx_msg;
835 u16 buf_size; /* queue buffer size */
836 u16 ring_size; /* Number of descriptors */
837 idpf_ctlq_reg_t reg; /* registers accessed by ctlqs */
840 /* PF/VF mailbox commands */
843 /* idpf_mbq_opc_send_msg_to_pf:
844 * usage: used by PF or VF to send a message to its CPF
845 * target: RX queue and function ID of parent PF taken from HW
847 idpf_mbq_opc_send_msg_to_pf = 0x0801,
849 /* idpf_mbq_opc_send_msg_to_vf:
850 * usage: used by PF to send message to a VF
851 * target: VF control queue ID must be specified in descriptor
853 idpf_mbq_opc_send_msg_to_vf = 0x0802,
855 /* idpf_mbq_opc_send_msg_to_peer_pf:
856 * usage: used by any function to send message to any peer PF
857 * target: RX queue and host of parent PF taken from HW
859 idpf_mbq_opc_send_msg_to_peer_pf = 0x0803,
861 /* idpf_mbq_opc_send_msg_to_peer_drv:
862 * usage: used by any function to send message to any peer driver
863 * target: RX queue and target host must be specific in descriptor
865 idpf_mbq_opc_send_msg_to_peer_drv = 0x0804,
872 u16 datalen; /* 0 for direct commands */
900 int idpf_ctlq_init (vlib_main_t *vm, idpf_device_t *id, u8 num_q,
901 idpf_ctlq_create_info_t *q_info);
902 int idpf_ctlq_add (vlib_main_t *vm, idpf_device_t *id,
903 idpf_ctlq_create_info_t *qinfo, struct idpf_ctlq_info **cq);
904 void idpf_ctlq_remove (idpf_device_t *id, struct idpf_ctlq_info *cq);
905 int idpf_ctlq_send (idpf_device_t *id, struct idpf_ctlq_info *cq,
906 u16 num_q_msg, idpf_ctlq_msg_t q_msg[]);
907 int idpf_ctlq_recv (struct idpf_ctlq_info *cq, u16 *num_q_msg,
908 idpf_ctlq_msg_t *q_msg);
909 int idpf_ctlq_clean_sq (struct idpf_ctlq_info *cq, u16 *clean_count,
910 idpf_ctlq_msg_t *msg_status[]);
911 int idpf_ctlq_post_rx_buffs (idpf_device_t *id, struct idpf_ctlq_info *cq,
912 u16 *buff_count, idpf_dma_mem_t **buffs);
913 void idpf_ctlq_deinit (idpf_device_t *id);
914 int idpf_ctlq_alloc_ring_res (vlib_main_t *vm, idpf_device_t *id,
915 struct idpf_ctlq_info *cq);
916 void idpf_ctlq_dealloc_ring_res (idpf_device_t *id, struct idpf_ctlq_info *cq);
917 void *idpf_alloc_dma_mem (vlib_main_t *vm, idpf_device_t *id,
918 idpf_dma_mem_t *mem, u64 size);
919 void idpf_free_dma_mem (idpf_device_t *id, idpf_dma_mem_t *mem);
924 * fd.io coding-style-patch-verification: ON
927 * eval: (c-set-style "gnu")