drivers/net/i40e/i40e_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2017 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdio.h>
  35 #include <errno.h>
  36 #include <stdint.h>
  37 #include <string.h>
  38 #include <unistd.h>
  39 #include <stdarg.h>
  40 #include <inttypes.h>
  41 #include <assert.h>
  42
  43 #include <rte_string_fns.h>
  44 #include <rte_pci.h>
  45 #include <rte_ether.h>
  46 #include <rte_ethdev.h>
  47 #include <rte_memzone.h>
  48 #include <rte_malloc.h>
  49 #include <rte_memcpy.h>
  50 #include <rte_alarm.h>
  51 #include <rte_dev.h>
  52 #include <rte_eth_ctrl.h>
  53 #include <rte_tailq.h>
  54
  55 #include "i40e_logs.h"
  56 #include "base/i40e_prototype.h"
  57 #include "base/i40e_adminq_cmd.h"
  58 #include "base/i40e_type.h"
  59 #include "base/i40e_register.h"
  60 #include "base/i40e_dcb.h"
  61 #include "i40e_ethdev.h"
  62 #include "i40e_rxtx.h"
  63 #include "i40e_pf.h"
  64 #include "i40e_regs.h"
  65
  66 #define ETH_I40E_FLOATING_VEB_ARG       "enable_floating_veb"
  67 #define ETH_I40E_FLOATING_VEB_LIST_ARG  "floating_veb_list"
  68
  69 #define I40E_CLEAR_PXE_WAIT_MS     200
  70
  71 /* Maximun number of capability elements */
  72 #define I40E_MAX_CAP_ELE_NUM       128
  73
  74 /* Wait count and inteval */
  75 #define I40E_CHK_Q_ENA_COUNT       1000
  76 #define I40E_CHK_Q_ENA_INTERVAL_US 1000
  77
  78 /* Maximun number of VSI */
  79 #define I40E_MAX_NUM_VSIS          (384UL)
  80
  81 #define I40E_PRE_TX_Q_CFG_WAIT_US       10 /* 10 us */
  82
  83 /* Flow control default timer */
  84 #define I40E_DEFAULT_PAUSE_TIME 0xFFFFU
  85
  86 /* Flow control enable fwd bit */
  87 #define I40E_PRTMAC_FWD_CTRL   0x00000001
  88
  89 /* Receive Packet Buffer size */
  90 #define I40E_RXPBSIZE (968 * 1024)
  91
  92 /* Kilobytes shift */
  93 #define I40E_KILOSHIFT 10
  94
  95 /* Flow control default high water */
  96 #define I40E_DEFAULT_HIGH_WATER (0xF2000 >> I40E_KILOSHIFT)
  97
  98 /* Flow control default low water */
  99 #define I40E_DEFAULT_LOW_WATER  (0xF2000 >> I40E_KILOSHIFT)
 100
 101 /* Receive Average Packet Size in Byte*/
 102 #define I40E_PACKET_AVERAGE_SIZE 128
 103
 104 /* Mask of PF interrupt causes */
 105 #define I40E_PFINT_ICR0_ENA_MASK ( \
 106                 I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | \
 107                 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | \
 108                 I40E_PFINT_ICR0_ENA_GRST_MASK | \
 109                 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | \
 110                 I40E_PFINT_ICR0_ENA_STORM_DETECT_MASK | \
 111                 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | \
 112                 I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK | \
 113                 I40E_PFINT_ICR0_ENA_VFLR_MASK | \
 114                 I40E_PFINT_ICR0_ENA_ADMINQ_MASK)
 115
 116 #define I40E_FLOW_TYPES ( \
 117         (1UL << RTE_ETH_FLOW_FRAG_IPV4) | \
 118         (1UL << RTE_ETH_FLOW_NONFRAG_IPV4_TCP) | \
 119         (1UL << RTE_ETH_FLOW_NONFRAG_IPV4_UDP) | \
 120         (1UL << RTE_ETH_FLOW_NONFRAG_IPV4_SCTP) | \
 121         (1UL << RTE_ETH_FLOW_NONFRAG_IPV4_OTHER) | \
 122         (1UL << RTE_ETH_FLOW_FRAG_IPV6) | \
 123         (1UL << RTE_ETH_FLOW_NONFRAG_IPV6_TCP) | \
 124         (1UL << RTE_ETH_FLOW_NONFRAG_IPV6_UDP) | \
 125         (1UL << RTE_ETH_FLOW_NONFRAG_IPV6_SCTP) | \
 126         (1UL << RTE_ETH_FLOW_NONFRAG_IPV6_OTHER) | \
 127         (1UL << RTE_ETH_FLOW_L2_PAYLOAD))
 128
 129 /* Additional timesync values. */
 130 #define I40E_PTP_40GB_INCVAL     0x0199999999ULL
 131 #define I40E_PTP_10GB_INCVAL     0x0333333333ULL
 132 #define I40E_PTP_1GB_INCVAL      0x2000000000ULL
 133 #define I40E_PRTTSYN_TSYNENA     0x80000000
 134 #define I40E_PRTTSYN_TSYNTYPE    0x0e000000
 135 #define I40E_CYCLECOUNTER_MASK   0xffffffffffffffffULL
 136
 137 #define I40E_MAX_PERCENT            100
 138 #define I40E_DEFAULT_DCB_APP_NUM    1
 139 #define I40E_DEFAULT_DCB_APP_PRIO   3
 140
 141 #define I40E_INSET_NONE            0x00000000000000000ULL
 142
 143 /* bit0 ~ bit 7 */
 144 #define I40E_INSET_DMAC            0x0000000000000001ULL
 145 #define I40E_INSET_SMAC            0x0000000000000002ULL
 146 #define I40E_INSET_VLAN_OUTER      0x0000000000000004ULL
 147 #define I40E_INSET_VLAN_INNER      0x0000000000000008ULL
 148 #define I40E_INSET_VLAN_TUNNEL     0x0000000000000010ULL
 149
 150 /* bit 8 ~ bit 15 */
 151 #define I40E_INSET_IPV4_SRC        0x0000000000000100ULL
 152 #define I40E_INSET_IPV4_DST        0x0000000000000200ULL
 153 #define I40E_INSET_IPV6_SRC        0x0000000000000400ULL
 154 #define I40E_INSET_IPV6_DST        0x0000000000000800ULL
 155 #define I40E_INSET_SRC_PORT        0x0000000000001000ULL
 156 #define I40E_INSET_DST_PORT        0x0000000000002000ULL
 157 #define I40E_INSET_SCTP_VT         0x0000000000004000ULL
 158
 159 /* bit 16 ~ bit 31 */
 160 #define I40E_INSET_IPV4_TOS        0x0000000000010000ULL
 161 #define I40E_INSET_IPV4_PROTO      0x0000000000020000ULL
 162 #define I40E_INSET_IPV4_TTL        0x0000000000040000ULL
 163 #define I40E_INSET_IPV6_TC         0x0000000000080000ULL
 164 #define I40E_INSET_IPV6_FLOW       0x0000000000100000ULL
 165 #define I40E_INSET_IPV6_NEXT_HDR   0x0000000000200000ULL
 166 #define I40E_INSET_IPV6_HOP_LIMIT  0x0000000000400000ULL
 167 #define I40E_INSET_TCP_FLAGS       0x0000000000800000ULL
 168
 169 /* bit 32 ~ bit 47, tunnel fields */
 170 #define I40E_INSET_TUNNEL_IPV4_DST       0x0000000100000000ULL
 171 #define I40E_INSET_TUNNEL_IPV6_DST       0x0000000200000000ULL
 172 #define I40E_INSET_TUNNEL_DMAC           0x0000000400000000ULL
 173 #define I40E_INSET_TUNNEL_SRC_PORT       0x0000000800000000ULL
 174 #define I40E_INSET_TUNNEL_DST_PORT       0x0000001000000000ULL
 175 #define I40E_INSET_TUNNEL_ID             0x0000002000000000ULL
 176
 177 /* bit 48 ~ bit 55 */
 178 #define I40E_INSET_LAST_ETHER_TYPE 0x0001000000000000ULL
 179
 180 /* bit 56 ~ bit 63, Flex Payload */
 181 #define I40E_INSET_FLEX_PAYLOAD_W1 0x0100000000000000ULL
 182 #define I40E_INSET_FLEX_PAYLOAD_W2 0x0200000000000000ULL
 183 #define I40E_INSET_FLEX_PAYLOAD_W3 0x0400000000000000ULL
 184 #define I40E_INSET_FLEX_PAYLOAD_W4 0x0800000000000000ULL
 185 #define I40E_INSET_FLEX_PAYLOAD_W5 0x1000000000000000ULL
 186 #define I40E_INSET_FLEX_PAYLOAD_W6 0x2000000000000000ULL
 187 #define I40E_INSET_FLEX_PAYLOAD_W7 0x4000000000000000ULL
 188 #define I40E_INSET_FLEX_PAYLOAD_W8 0x8000000000000000ULL
 189 #define I40E_INSET_FLEX_PAYLOAD \
 190         (I40E_INSET_FLEX_PAYLOAD_W1 | I40E_INSET_FLEX_PAYLOAD_W2 | \
 191         I40E_INSET_FLEX_PAYLOAD_W3 | I40E_INSET_FLEX_PAYLOAD_W4 | \
 192         I40E_INSET_FLEX_PAYLOAD_W5 | I40E_INSET_FLEX_PAYLOAD_W6 | \
 193         I40E_INSET_FLEX_PAYLOAD_W7 | I40E_INSET_FLEX_PAYLOAD_W8)
 194
 195 /**
 196  * Below are values for writing un-exposed registers suggested
 197  * by silicon experts
 198  */
 199 /* Destination MAC address */
 200 #define I40E_REG_INSET_L2_DMAC                   0xE000000000000000ULL
 201 /* Source MAC address */
 202 #define I40E_REG_INSET_L2_SMAC                   0x1C00000000000000ULL
 203 /* Outer (S-Tag) VLAN tag in the outer L2 header */
 204 #define I40E_REG_INSET_L2_OUTER_VLAN             0x0000000004000000ULL
 205 /* Inner (C-Tag) or single VLAN tag in the outer L2 header */
 206 #define I40E_REG_INSET_L2_INNER_VLAN             0x0080000000000000ULL
 207 /* Single VLAN tag in the inner L2 header */
 208 #define I40E_REG_INSET_TUNNEL_VLAN               0x0100000000000000ULL
 209 /* Source IPv4 address */
 210 #define I40E_REG_INSET_L3_SRC_IP4                0x0001800000000000ULL
 211 /* Destination IPv4 address */
 212 #define I40E_REG_INSET_L3_DST_IP4                0x0000001800000000ULL
 213 /* Source IPv4 address for X722 */
 214 #define I40E_X722_REG_INSET_L3_SRC_IP4           0x0006000000000000ULL
 215 /* Destination IPv4 address for X722 */
 216 #define I40E_X722_REG_INSET_L3_DST_IP4           0x0000060000000000ULL
 217 /* IPv4 Protocol for X722 */
 218 #define I40E_X722_REG_INSET_L3_IP4_PROTO         0x0010000000000000ULL
 219 /* IPv4 Time to Live for X722 */
 220 #define I40E_X722_REG_INSET_L3_IP4_TTL           0x0010000000000000ULL
 221 /* IPv4 Type of Service (TOS) */
 222 #define I40E_REG_INSET_L3_IP4_TOS                0x0040000000000000ULL
 223 /* IPv4 Protocol */
 224 #define I40E_REG_INSET_L3_IP4_PROTO              0x0004000000000000ULL
 225 /* IPv4 Time to Live */
 226 #define I40E_REG_INSET_L3_IP4_TTL                0x0004000000000000ULL
 227 /* Source IPv6 address */
 228 #define I40E_REG_INSET_L3_SRC_IP6                0x0007F80000000000ULL
 229 /* Destination IPv6 address */
 230 #define I40E_REG_INSET_L3_DST_IP6                0x000007F800000000ULL
 231 /* IPv6 Traffic Class (TC) */
 232 #define I40E_REG_INSET_L3_IP6_TC                 0x0040000000000000ULL
 233 /* IPv6 Next Header */
 234 #define I40E_REG_INSET_L3_IP6_NEXT_HDR           0x0008000000000000ULL
 235 /* IPv6 Hop Limit */
 236 #define I40E_REG_INSET_L3_IP6_HOP_LIMIT          0x0008000000000000ULL
 237 /* Source L4 port */
 238 #define I40E_REG_INSET_L4_SRC_PORT               0x0000000400000000ULL
 239 /* Destination L4 port */
 240 #define I40E_REG_INSET_L4_DST_PORT               0x0000000200000000ULL
 241 /* SCTP verification tag */
 242 #define I40E_REG_INSET_L4_SCTP_VERIFICATION_TAG  0x0000000180000000ULL
 243 /* Inner destination MAC address (MAC-in-UDP/MAC-in-GRE)*/
 244 #define I40E_REG_INSET_TUNNEL_L2_INNER_DST_MAC   0x0000000001C00000ULL
 245 /* Source port of tunneling UDP */
 246 #define I40E_REG_INSET_TUNNEL_L4_UDP_SRC_PORT    0x0000000000200000ULL
 247 /* Destination port of tunneling UDP */
 248 #define I40E_REG_INSET_TUNNEL_L4_UDP_DST_PORT    0x0000000000100000ULL
 249 /* UDP Tunneling ID, NVGRE/GRE key */
 250 #define I40E_REG_INSET_TUNNEL_ID                 0x00000000000C0000ULL
 251 /* Last ether type */
 252 #define I40E_REG_INSET_LAST_ETHER_TYPE           0x0000000000004000ULL
 253 /* Tunneling outer destination IPv4 address */
 254 #define I40E_REG_INSET_TUNNEL_L3_DST_IP4         0x00000000000000C0ULL
 255 /* Tunneling outer destination IPv6 address */
 256 #define I40E_REG_INSET_TUNNEL_L3_DST_IP6         0x0000000000003FC0ULL
 257 /* 1st word of flex payload */
 258 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD1        0x0000000000002000ULL
 259 /* 2nd word of flex payload */
 260 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD2        0x0000000000001000ULL
 261 /* 3rd word of flex payload */
 262 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD3        0x0000000000000800ULL
 263 /* 4th word of flex payload */
 264 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD4        0x0000000000000400ULL
 265 /* 5th word of flex payload */
 266 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD5        0x0000000000000200ULL
 267 /* 6th word of flex payload */
 268 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD6        0x0000000000000100ULL
 269 /* 7th word of flex payload */
 270 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD7        0x0000000000000080ULL
 271 /* 8th word of flex payload */
 272 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD8        0x0000000000000040ULL
 273 /* all 8 words flex payload */
 274 #define I40E_REG_INSET_FLEX_PAYLOAD_WORDS        0x0000000000003FC0ULL
 275 #define I40E_REG_INSET_MASK_DEFAULT              0x0000000000000000ULL
 276
 277 #define I40E_TRANSLATE_INSET 0
 278 #define I40E_TRANSLATE_REG   1
 279
 280 #define I40E_INSET_IPV4_TOS_MASK        0x0009FF00UL
 281 #define I40E_INSET_IPv4_TTL_MASK        0x000D00FFUL
 282 #define I40E_INSET_IPV4_PROTO_MASK      0x000DFF00UL
 283 #define I40E_INSET_IPV6_TC_MASK         0x0009F00FUL
 284 #define I40E_INSET_IPV6_HOP_LIMIT_MASK  0x000CFF00UL
 285 #define I40E_INSET_IPV6_NEXT_HDR_MASK   0x000C00FFUL
 286
 287 #define I40E_GL_SWT_L2TAGCTRL(_i)             (0x001C0A70 + ((_i) * 4))
 288 #define I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_SHIFT 16
 289 #define I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_MASK  \
 290         I40E_MASK(0xFFFF, I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_SHIFT)
 291
 292 /* PCI offset for querying capability */
 293 #define PCI_DEV_CAP_REG            0xA4
 294 /* PCI offset for enabling/disabling Extended Tag */
 295 #define PCI_DEV_CTRL_REG           0xA8
 296 /* Bit mask of Extended Tag capability */
 297 #define PCI_DEV_CAP_EXT_TAG_MASK   0x20
 298 /* Bit shift of Extended Tag enable/disable */
 299 #define PCI_DEV_CTRL_EXT_TAG_SHIFT 8
 300 /* Bit mask of Extended Tag enable/disable */
 301 #define PCI_DEV_CTRL_EXT_TAG_MASK  (1 << PCI_DEV_CTRL_EXT_TAG_SHIFT)
 302
 303 static int eth_i40e_dev_init(struct rte_eth_dev *eth_dev);
 304 static int eth_i40e_dev_uninit(struct rte_eth_dev *eth_dev);
 305 static int i40e_dev_configure(struct rte_eth_dev *dev);
 306 static int i40e_dev_start(struct rte_eth_dev *dev);
 307 static void i40e_dev_stop(struct rte_eth_dev *dev);
 308 static void i40e_dev_close(struct rte_eth_dev *dev);
 309 static void i40e_dev_promiscuous_enable(struct rte_eth_dev *dev);
 310 static void i40e_dev_promiscuous_disable(struct rte_eth_dev *dev);
 311 static void i40e_dev_allmulticast_enable(struct rte_eth_dev *dev);
 312 static void i40e_dev_allmulticast_disable(struct rte_eth_dev *dev);
 313 static int i40e_dev_set_link_up(struct rte_eth_dev *dev);
 314 static int i40e_dev_set_link_down(struct rte_eth_dev *dev);
 315 static void i40e_dev_stats_get(struct rte_eth_dev *dev,
 316                                struct rte_eth_stats *stats);
 317 static int i40e_dev_xstats_get(struct rte_eth_dev *dev,
 318                                struct rte_eth_xstat *xstats, unsigned n);
 319 static int i40e_dev_xstats_get_names(struct rte_eth_dev *dev,
 320                                      struct rte_eth_xstat_name *xstats_names,
 321                                      unsigned limit);
 322 static void i40e_dev_stats_reset(struct rte_eth_dev *dev);
 323 static int i40e_dev_queue_stats_mapping_set(struct rte_eth_dev *dev,
 324                                             uint16_t queue_id,
 325                                             uint8_t stat_idx,
 326                                             uint8_t is_rx);
 327 static void i40e_dev_info_get(struct rte_eth_dev *dev,
 328                               struct rte_eth_dev_info *dev_info);
 329 static int i40e_vlan_filter_set(struct rte_eth_dev *dev,
 330                                 uint16_t vlan_id,
 331                                 int on);
 332 static int i40e_vlan_tpid_set(struct rte_eth_dev *dev,
 333                               enum rte_vlan_type vlan_type,
 334                               uint16_t tpid);
 335 static void i40e_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 336 static void i40e_vlan_strip_queue_set(struct rte_eth_dev *dev,
 337                                       uint16_t queue,
 338                                       int on);
 339 static int i40e_vlan_pvid_set(struct rte_eth_dev *dev, uint16_t pvid, int on);
 340 static int i40e_dev_led_on(struct rte_eth_dev *dev);
 341 static int i40e_dev_led_off(struct rte_eth_dev *dev);
 342 static int i40e_flow_ctrl_get(struct rte_eth_dev *dev,
 343                               struct rte_eth_fc_conf *fc_conf);
 344 static int i40e_flow_ctrl_set(struct rte_eth_dev *dev,
 345                               struct rte_eth_fc_conf *fc_conf);
 346 static int i40e_priority_flow_ctrl_set(struct rte_eth_dev *dev,
 347                                        struct rte_eth_pfc_conf *pfc_conf);
 348 static void i40e_macaddr_add(struct rte_eth_dev *dev,
 349                           struct ether_addr *mac_addr,
 350                           uint32_t index,
 351                           uint32_t pool);
 352 static void i40e_macaddr_remove(struct rte_eth_dev *dev, uint32_t index);
 353 static int i40e_dev_rss_reta_update(struct rte_eth_dev *dev,
 354                                     struct rte_eth_rss_reta_entry64 *reta_conf,
 355                                     uint16_t reta_size);
 356 static int i40e_dev_rss_reta_query(struct rte_eth_dev *dev,
 357                                    struct rte_eth_rss_reta_entry64 *reta_conf,
 358                                    uint16_t reta_size);
 359
 360 static int i40e_get_cap(struct i40e_hw *hw);
 361 static int i40e_pf_parameter_init(struct rte_eth_dev *dev);
 362 static int i40e_pf_setup(struct i40e_pf *pf);
 363 static int i40e_dev_rxtx_init(struct i40e_pf *pf);
 364 static int i40e_vmdq_setup(struct rte_eth_dev *dev);
 365 static int i40e_dcb_init_configure(struct rte_eth_dev *dev, bool sw_dcb);
 366 static int i40e_dcb_setup(struct rte_eth_dev *dev);
 367 static void i40e_stat_update_32(struct i40e_hw *hw, uint32_t reg,
 368                 bool offset_loaded, uint64_t *offset, uint64_t *stat);
 369 static void i40e_stat_update_48(struct i40e_hw *hw,
 370                                uint32_t hireg,
 371                                uint32_t loreg,
 372                                bool offset_loaded,
 373                                uint64_t *offset,
 374                                uint64_t *stat);
 375 static void i40e_pf_config_irq0(struct i40e_hw *hw, bool no_queue);
 376 static void i40e_dev_interrupt_handler(
 377                 __rte_unused struct rte_intr_handle *handle, void *param);
 378 static int i40e_res_pool_init(struct i40e_res_pool_info *pool,
 379                                 uint32_t base, uint32_t num);
 380 static void i40e_res_pool_destroy(struct i40e_res_pool_info *pool);
 381 static int i40e_res_pool_free(struct i40e_res_pool_info *pool,
 382                         uint32_t base);
 383 static int i40e_res_pool_alloc(struct i40e_res_pool_info *pool,
 384                         uint16_t num);
 385 static int i40e_dev_init_vlan(struct rte_eth_dev *dev);
 386 static int i40e_veb_release(struct i40e_veb *veb);
 387 static struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf,
 388                                                 struct i40e_vsi *vsi);
 389 static int i40e_pf_config_mq_rx(struct i40e_pf *pf);
 390 static int i40e_vsi_config_double_vlan(struct i40e_vsi *vsi, int on);
 391 static inline int i40e_find_all_vlan_for_mac(struct i40e_vsi *vsi,
 392                                              struct i40e_macvlan_filter *mv_f,
 393                                              int num,
 394                                              struct ether_addr *addr);
 395 static inline int i40e_find_all_mac_for_vlan(struct i40e_vsi *vsi,
 396                                              struct i40e_macvlan_filter *mv_f,
 397                                              int num,
 398                                              uint16_t vlan);
 399 static int i40e_vsi_remove_all_macvlan_filter(struct i40e_vsi *vsi);
 400 static int i40e_dev_rss_hash_update(struct rte_eth_dev *dev,
 401                                     struct rte_eth_rss_conf *rss_conf);
 402 static int i40e_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
 403                                       struct rte_eth_rss_conf *rss_conf);
 404 static int i40e_dev_udp_tunnel_port_add(struct rte_eth_dev *dev,
 405                                         struct rte_eth_udp_tunnel *udp_tunnel);
 406 static int i40e_dev_udp_tunnel_port_del(struct rte_eth_dev *dev,
 407                                         struct rte_eth_udp_tunnel *udp_tunnel);
 408 static void i40e_filter_input_set_init(struct i40e_pf *pf);
 409 static int i40e_ethertype_filter_set(struct i40e_pf *pf,
 410                         struct rte_eth_ethertype_filter *filter,
 411                         bool add);
 412 static int i40e_ethertype_filter_handle(struct rte_eth_dev *dev,
 413                                 enum rte_filter_op filter_op,
 414                                 void *arg);
 415 static int i40e_dev_filter_ctrl(struct rte_eth_dev *dev,
 416                                 enum rte_filter_type filter_type,
 417                                 enum rte_filter_op filter_op,
 418                                 void *arg);
 419 static int i40e_dev_get_dcb_info(struct rte_eth_dev *dev,
 420                                   struct rte_eth_dcb_info *dcb_info);
 421 static int i40e_dev_sync_phy_type(struct i40e_hw *hw);
 422 static void i40e_configure_registers(struct i40e_hw *hw);
 423 static void i40e_hw_init(struct rte_eth_dev *dev);
 424 static int i40e_config_qinq(struct i40e_hw *hw, struct i40e_vsi *vsi);
 425 static enum i40e_status_code i40e_aq_del_mirror_rule(struct i40e_hw *hw,
 426                                                      uint16_t seid,
 427                                                      uint16_t rule_type,
 428                                                      uint16_t *entries,
 429                                                      uint16_t count,
 430                                                      uint16_t rule_id);
 431 static int i40e_mirror_rule_set(struct rte_eth_dev *dev,
 432                         struct rte_eth_mirror_conf *mirror_conf,
 433                         uint8_t sw_id, uint8_t on);
 434 static int i40e_mirror_rule_reset(struct rte_eth_dev *dev, uint8_t sw_id);
 435
 436 static int i40e_timesync_enable(struct rte_eth_dev *dev);
 437 static int i40e_timesync_disable(struct rte_eth_dev *dev);
 438 static int i40e_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
 439                                            struct timespec *timestamp,
 440                                            uint32_t flags);
 441 static int i40e_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
 442                                            struct timespec *timestamp);
 443 static void i40e_read_stats_registers(struct i40e_pf *pf, struct i40e_hw *hw);
 444
 445 static int i40e_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta);
 446
 447 static int i40e_timesync_read_time(struct rte_eth_dev *dev,
 448                                    struct timespec *timestamp);
 449 static int i40e_timesync_write_time(struct rte_eth_dev *dev,
 450                                     const struct timespec *timestamp);
 451
 452 static int i40e_dev_rx_queue_intr_enable(struct rte_eth_dev *dev,
 453                                          uint16_t queue_id);
 454 static int i40e_dev_rx_queue_intr_disable(struct rte_eth_dev *dev,
 455                                           uint16_t queue_id);
 456
 457 static int i40e_get_regs(struct rte_eth_dev *dev,
 458                          struct rte_dev_reg_info *regs);
 459
 460 static int i40e_get_eeprom_length(struct rte_eth_dev *dev);
 461
 462 static int i40e_get_eeprom(struct rte_eth_dev *dev,
 463                            struct rte_dev_eeprom_info *eeprom);
 464
 465 static void i40e_set_default_mac_addr(struct rte_eth_dev *dev,
 466                                       struct ether_addr *mac_addr);
 467
 468 static int i40e_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
 469 static void i40e_notify_all_vfs_link_status(struct rte_eth_dev *dev);
 470
 471 static const struct rte_pci_id pci_id_i40e_map[] = {
 472         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_XL710) },
 473         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QEMU) },
 474         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_B) },
 475         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_C) },
 476         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_A) },
 477         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_B) },
 478         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_C) },
 479         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T) },
 480         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_20G_KR2) },
 481         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_20G_KR2_A) },
 482         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T4) },
 483         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_25G_B) },
 484         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_25G_SFP28) },
 485         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_X722_A0) },
 486         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_X722) },
 487         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_X722) },
 488         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_X722) },
 489         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_1G_BASE_T_X722) },
 490         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T_X722) },
 491         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_I_X722) },
 492         { .vendor_id = 0, /* sentinel */ },
 493 };
 494
 495 static const struct eth_dev_ops i40e_eth_dev_ops = {
 496         .dev_configure                = i40e_dev_configure,
 497         .dev_start                    = i40e_dev_start,
 498         .dev_stop                     = i40e_dev_stop,
 499         .dev_close                    = i40e_dev_close,
 500         .promiscuous_enable           = i40e_dev_promiscuous_enable,
 501         .promiscuous_disable          = i40e_dev_promiscuous_disable,
 502         .allmulticast_enable          = i40e_dev_allmulticast_enable,
 503         .allmulticast_disable         = i40e_dev_allmulticast_disable,
 504         .dev_set_link_up              = i40e_dev_set_link_up,
 505         .dev_set_link_down            = i40e_dev_set_link_down,
 506         .link_update                  = i40e_dev_link_update,
 507         .stats_get                    = i40e_dev_stats_get,
 508         .xstats_get                   = i40e_dev_xstats_get,
 509         .xstats_get_names             = i40e_dev_xstats_get_names,
 510         .stats_reset                  = i40e_dev_stats_reset,
 511         .xstats_reset                 = i40e_dev_stats_reset,
 512         .queue_stats_mapping_set      = i40e_dev_queue_stats_mapping_set,
 513         .dev_infos_get                = i40e_dev_info_get,
 514         .dev_supported_ptypes_get     = i40e_dev_supported_ptypes_get,
 515         .vlan_filter_set              = i40e_vlan_filter_set,
 516         .vlan_tpid_set                = i40e_vlan_tpid_set,
 517         .vlan_offload_set             = i40e_vlan_offload_set,
 518         .vlan_strip_queue_set         = i40e_vlan_strip_queue_set,
 519         .vlan_pvid_set                = i40e_vlan_pvid_set,
 520         .rx_queue_start               = i40e_dev_rx_queue_start,
 521         .rx_queue_stop                = i40e_dev_rx_queue_stop,
 522         .tx_queue_start               = i40e_dev_tx_queue_start,
 523         .tx_queue_stop                = i40e_dev_tx_queue_stop,
 524         .rx_queue_setup               = i40e_dev_rx_queue_setup,
 525         .rx_queue_intr_enable         = i40e_dev_rx_queue_intr_enable,
 526         .rx_queue_intr_disable        = i40e_dev_rx_queue_intr_disable,
 527         .rx_queue_release             = i40e_dev_rx_queue_release,
 528         .rx_queue_count               = i40e_dev_rx_queue_count,
 529         .rx_descriptor_done           = i40e_dev_rx_descriptor_done,
 530         .tx_queue_setup               = i40e_dev_tx_queue_setup,
 531         .tx_queue_release             = i40e_dev_tx_queue_release,
 532         .dev_led_on                   = i40e_dev_led_on,
 533         .dev_led_off                  = i40e_dev_led_off,
 534         .flow_ctrl_get                = i40e_flow_ctrl_get,
 535         .flow_ctrl_set                = i40e_flow_ctrl_set,
 536         .priority_flow_ctrl_set       = i40e_priority_flow_ctrl_set,
 537         .mac_addr_add                 = i40e_macaddr_add,
 538         .mac_addr_remove              = i40e_macaddr_remove,
 539         .reta_update                  = i40e_dev_rss_reta_update,
 540         .reta_query                   = i40e_dev_rss_reta_query,
 541         .rss_hash_update              = i40e_dev_rss_hash_update,
 542         .rss_hash_conf_get            = i40e_dev_rss_hash_conf_get,
 543         .udp_tunnel_port_add          = i40e_dev_udp_tunnel_port_add,
 544         .udp_tunnel_port_del          = i40e_dev_udp_tunnel_port_del,
 545         .filter_ctrl                  = i40e_dev_filter_ctrl,
 546         .rxq_info_get                 = i40e_rxq_info_get,
 547         .txq_info_get                 = i40e_txq_info_get,
 548         .mirror_rule_set              = i40e_mirror_rule_set,
 549         .mirror_rule_reset            = i40e_mirror_rule_reset,
 550         .timesync_enable              = i40e_timesync_enable,
 551         .timesync_disable             = i40e_timesync_disable,
 552         .timesync_read_rx_timestamp   = i40e_timesync_read_rx_timestamp,
 553         .timesync_read_tx_timestamp   = i40e_timesync_read_tx_timestamp,
 554         .get_dcb_info                 = i40e_dev_get_dcb_info,
 555         .timesync_adjust_time         = i40e_timesync_adjust_time,
 556         .timesync_read_time           = i40e_timesync_read_time,
 557         .timesync_write_time          = i40e_timesync_write_time,
 558         .get_reg                      = i40e_get_regs,
 559         .get_eeprom_length            = i40e_get_eeprom_length,
 560         .get_eeprom                   = i40e_get_eeprom,
 561         .mac_addr_set                 = i40e_set_default_mac_addr,
 562         .mtu_set                      = i40e_dev_mtu_set,
 563 };
 564
 565 /* store statistics names and its offset in stats structure */
 566 struct rte_i40e_xstats_name_off {
 567         char name[RTE_ETH_XSTATS_NAME_SIZE];
 568         unsigned offset;
 569 };
 570
 571 static const struct rte_i40e_xstats_name_off rte_i40e_stats_strings[] = {
 572         {"rx_unicast_packets", offsetof(struct i40e_eth_stats, rx_unicast)},
 573         {"rx_multicast_packets", offsetof(struct i40e_eth_stats, rx_multicast)},
 574         {"rx_broadcast_packets", offsetof(struct i40e_eth_stats, rx_broadcast)},
 575         {"rx_dropped", offsetof(struct i40e_eth_stats, rx_discards)},
 576         {"rx_unknown_protocol_packets", offsetof(struct i40e_eth_stats,
 577                 rx_unknown_protocol)},
 578         {"tx_unicast_packets", offsetof(struct i40e_eth_stats, tx_unicast)},
 579         {"tx_multicast_packets", offsetof(struct i40e_eth_stats, tx_multicast)},
 580         {"tx_broadcast_packets", offsetof(struct i40e_eth_stats, tx_broadcast)},
 581         {"tx_dropped", offsetof(struct i40e_eth_stats, tx_discards)},
 582 };
 583
 584 #define I40E_NB_ETH_XSTATS (sizeof(rte_i40e_stats_strings) / \
 585                 sizeof(rte_i40e_stats_strings[0]))
 586
 587 static const struct rte_i40e_xstats_name_off rte_i40e_hw_port_strings[] = {
 588         {"tx_link_down_dropped", offsetof(struct i40e_hw_port_stats,
 589                 tx_dropped_link_down)},
 590         {"rx_crc_errors", offsetof(struct i40e_hw_port_stats, crc_errors)},
 591         {"rx_illegal_byte_errors", offsetof(struct i40e_hw_port_stats,
 592                 illegal_bytes)},
 593         {"rx_error_bytes", offsetof(struct i40e_hw_port_stats, error_bytes)},
 594         {"mac_local_errors", offsetof(struct i40e_hw_port_stats,
 595                 mac_local_faults)},
 596         {"mac_remote_errors", offsetof(struct i40e_hw_port_stats,
 597                 mac_remote_faults)},
 598         {"rx_length_errors", offsetof(struct i40e_hw_port_stats,
 599                 rx_length_errors)},
 600         {"tx_xon_packets", offsetof(struct i40e_hw_port_stats, link_xon_tx)},
 601         {"rx_xon_packets", offsetof(struct i40e_hw_port_stats, link_xon_rx)},
 602         {"tx_xoff_packets", offsetof(struct i40e_hw_port_stats, link_xoff_tx)},
 603         {"rx_xoff_packets", offsetof(struct i40e_hw_port_stats, link_xoff_rx)},
 604         {"rx_size_64_packets", offsetof(struct i40e_hw_port_stats, rx_size_64)},
 605         {"rx_size_65_to_127_packets", offsetof(struct i40e_hw_port_stats,
 606                 rx_size_127)},
 607         {"rx_size_128_to_255_packets", offsetof(struct i40e_hw_port_stats,
 608                 rx_size_255)},
 609         {"rx_size_256_to_511_packets", offsetof(struct i40e_hw_port_stats,
 610                 rx_size_511)},
 611         {"rx_size_512_to_1023_packets", offsetof(struct i40e_hw_port_stats,
 612                 rx_size_1023)},
 613         {"rx_size_1024_to_1522_packets", offsetof(struct i40e_hw_port_stats,
 614                 rx_size_1522)},
 615         {"rx_size_1523_to_max_packets", offsetof(struct i40e_hw_port_stats,
 616                 rx_size_big)},
 617         {"rx_undersized_errors", offsetof(struct i40e_hw_port_stats,
 618                 rx_undersize)},
 619         {"rx_oversize_errors", offsetof(struct i40e_hw_port_stats,
 620                 rx_oversize)},
 621         {"rx_mac_short_dropped", offsetof(struct i40e_hw_port_stats,
 622                 mac_short_packet_dropped)},
 623         {"rx_fragmented_errors", offsetof(struct i40e_hw_port_stats,
 624                 rx_fragments)},
 625         {"rx_jabber_errors", offsetof(struct i40e_hw_port_stats, rx_jabber)},
 626         {"tx_size_64_packets", offsetof(struct i40e_hw_port_stats, tx_size_64)},
 627         {"tx_size_65_to_127_packets", offsetof(struct i40e_hw_port_stats,
 628                 tx_size_127)},
 629         {"tx_size_128_to_255_packets", offsetof(struct i40e_hw_port_stats,
 630                 tx_size_255)},
 631         {"tx_size_256_to_511_packets", offsetof(struct i40e_hw_port_stats,
 632                 tx_size_511)},
 633         {"tx_size_512_to_1023_packets", offsetof(struct i40e_hw_port_stats,
 634                 tx_size_1023)},
 635         {"tx_size_1024_to_1522_packets", offsetof(struct i40e_hw_port_stats,
 636                 tx_size_1522)},
 637         {"tx_size_1523_to_max_packets", offsetof(struct i40e_hw_port_stats,
 638                 tx_size_big)},
 639         {"rx_flow_director_atr_match_packets",
 640                 offsetof(struct i40e_hw_port_stats, fd_atr_match)},
 641         {"rx_flow_director_sb_match_packets",
 642                 offsetof(struct i40e_hw_port_stats, fd_sb_match)},
 643         {"tx_low_power_idle_status", offsetof(struct i40e_hw_port_stats,
 644                 tx_lpi_status)},
 645         {"rx_low_power_idle_status", offsetof(struct i40e_hw_port_stats,
 646                 rx_lpi_status)},
 647         {"tx_low_power_idle_count", offsetof(struct i40e_hw_port_stats,
 648                 tx_lpi_count)},
 649         {"rx_low_power_idle_count", offsetof(struct i40e_hw_port_stats,
 650                 rx_lpi_count)},
 651 };
 652
 653 #define I40E_NB_HW_PORT_XSTATS (sizeof(rte_i40e_hw_port_strings) / \
 654                 sizeof(rte_i40e_hw_port_strings[0]))
 655
 656 static const struct rte_i40e_xstats_name_off rte_i40e_rxq_prio_strings[] = {
 657         {"xon_packets", offsetof(struct i40e_hw_port_stats,
 658                 priority_xon_rx)},
 659         {"xoff_packets", offsetof(struct i40e_hw_port_stats,
 660                 priority_xoff_rx)},
 661 };
 662
 663 #define I40E_NB_RXQ_PRIO_XSTATS (sizeof(rte_i40e_rxq_prio_strings) / \
 664                 sizeof(rte_i40e_rxq_prio_strings[0]))
 665
 666 static const struct rte_i40e_xstats_name_off rte_i40e_txq_prio_strings[] = {
 667         {"xon_packets", offsetof(struct i40e_hw_port_stats,
 668                 priority_xon_tx)},
 669         {"xoff_packets", offsetof(struct i40e_hw_port_stats,
 670                 priority_xoff_tx)},
 671         {"xon_to_xoff_packets", offsetof(struct i40e_hw_port_stats,
 672                 priority_xon_2_xoff)},
 673 };
 674
 675 #define I40E_NB_TXQ_PRIO_XSTATS (sizeof(rte_i40e_txq_prio_strings) / \
 676                 sizeof(rte_i40e_txq_prio_strings[0]))
 677
 678 static struct eth_driver rte_i40e_pmd = {
 679         .pci_drv = {
 680                 .id_table = pci_id_i40e_map,
 681                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
 682                         RTE_PCI_DRV_DETACHABLE,
 683                 .probe = rte_eth_dev_pci_probe,
 684                 .remove = rte_eth_dev_pci_remove,
 685         },
 686         .eth_dev_init = eth_i40e_dev_init,
 687         .eth_dev_uninit = eth_i40e_dev_uninit,
 688         .dev_private_size = sizeof(struct i40e_adapter),
 689 };
 690
 691 static inline int
 692 rte_i40e_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 693                                      struct rte_eth_link *link)
 694 {
 695         struct rte_eth_link *dst = link;
 696         struct rte_eth_link *src = &(dev->data->dev_link);
 697
 698         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 699                                         *(uint64_t *)src) == 0)
 700                 return -1;
 701
 702         return 0;
 703 }
 704
 705 static inline int
 706 rte_i40e_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 707                                       struct rte_eth_link *link)
 708 {
 709         struct rte_eth_link *dst = &(dev->data->dev_link);
 710         struct rte_eth_link *src = link;
 711
 712         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 713                                         *(uint64_t *)src) == 0)
 714                 return -1;
 715
 716         return 0;
 717 }
 718
 719 static inline void
 720 i40e_write_global_rx_ctl(struct i40e_hw *hw, u32 reg_addr, u32 reg_val)
 721 {
 722         i40e_write_rx_ctl(hw, reg_addr, reg_val);
 723         PMD_DRV_LOG(DEBUG, "Global register 0x%08x is modified "
 724                     "with value 0x%08x",
 725                     reg_addr, reg_val);
 726 }
 727
 728 RTE_PMD_REGISTER_PCI(net_i40e, rte_i40e_pmd.pci_drv);
 729 RTE_PMD_REGISTER_PCI_TABLE(net_i40e, pci_id_i40e_map);
 730
 731 #ifndef I40E_GLQF_ORT
 732 #define I40E_GLQF_ORT(_i)    (0x00268900 + ((_i) * 4))
 733 #endif
 734 #ifndef I40E_GLQF_PIT
 735 #define I40E_GLQF_PIT(_i)    (0x00268C80 + ((_i) * 4))
 736 #endif
 737
 738 static inline void i40e_GLQF_reg_init(struct i40e_hw *hw)
 739 {
 740         /*
 741          * Force global configuration for flexible payload
 742          * to the first 16 bytes of the corresponding L2/L3/L4 paylod.
 743          * This should be removed from code once proper
 744          * configuration API is added to avoid configuration conflicts
 745          * between ports of the same device.
 746          */
 747         I40E_WRITE_GLB_REG(hw, I40E_GLQF_ORT(33), 0x000000E0);
 748         I40E_WRITE_GLB_REG(hw, I40E_GLQF_ORT(34), 0x000000E3);
 749         I40E_WRITE_GLB_REG(hw, I40E_GLQF_ORT(35), 0x000000E6);
 750         i40e_global_cfg_warning(I40E_WARNING_ENA_FLX_PLD);
 751
 752         /*
 753          * Initialize registers for parsing packet type of QinQ
 754          * This should be removed from code once proper
 755          * configuration API is added to avoid configuration conflicts
 756          * between ports of the same device.
 757          */
 758         I40E_WRITE_GLB_REG(hw, I40E_GLQF_ORT(40), 0x00000029);
 759         I40E_WRITE_GLB_REG(hw, I40E_GLQF_PIT(9), 0x00009420);
 760         i40e_global_cfg_warning(I40E_WARNING_QINQ_PARSER);
 761 }
 762
 763 static inline void i40e_config_automask(struct i40e_pf *pf)
 764 {
 765         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
 766         uint32_t val;
 767
 768         /* INTENA flag is not auto-cleared for interrupt */
 769         val = I40E_READ_REG(hw, I40E_GLINT_CTL);
 770         val |= I40E_GLINT_CTL_DIS_AUTOMASK_PF0_MASK |
 771                 I40E_GLINT_CTL_DIS_AUTOMASK_VF0_MASK;
 772
 773         /* If support multi-driver, PF will use INT0. */
 774         if (!pf->support_multi_driver)
 775                 val |= I40E_GLINT_CTL_DIS_AUTOMASK_N_MASK;
 776
 777         I40E_WRITE_REG(hw, I40E_GLINT_CTL, val);
 778 }
 779
 780 #define I40E_FLOW_CONTROL_ETHERTYPE  0x8808
 781
 782 /*
 783  * Add a ethertype filter to drop all flow control frames transmitted
 784  * from VSIs.
 785 */
 786 static void
 787 i40e_add_tx_flow_control_drop_filter(struct i40e_pf *pf)
 788 {
 789         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
 790         uint16_t flags = I40E_AQC_ADD_CONTROL_PACKET_FLAGS_IGNORE_MAC |
 791                         I40E_AQC_ADD_CONTROL_PACKET_FLAGS_DROP |
 792                         I40E_AQC_ADD_CONTROL_PACKET_FLAGS_TX;
 793         int ret;
 794
 795         ret = i40e_aq_add_rem_control_packet_filter(hw, NULL,
 796                                 I40E_FLOW_CONTROL_ETHERTYPE, flags,
 797                                 pf->main_vsi_seid, 0,
 798                                 TRUE, NULL, NULL);
 799         if (ret)
 800                 PMD_INIT_LOG(ERR, "Failed to add filter to drop flow control "
 801                                   " frames from VSIs.");
 802 }
 803
 804 static int
 805 floating_veb_list_handler(__rte_unused const char *key,
 806                           const char *floating_veb_value,
 807                           void *opaque)
 808 {
 809         int idx = 0;
 810         unsigned int count = 0;
 811         char *end = NULL;
 812         int min, max;
 813         bool *vf_floating_veb = opaque;
 814
 815         while (isblank(*floating_veb_value))
 816                 floating_veb_value++;
 817
 818         /* Reset floating VEB configuration for VFs */
 819         for (idx = 0; idx < I40E_MAX_VF; idx++)
 820                 vf_floating_veb[idx] = false;
 821
 822         min = I40E_MAX_VF;
 823         do {
 824                 while (isblank(*floating_veb_value))
 825                         floating_veb_value++;
 826                 if (*floating_veb_value == '\0')
 827                         return -1;
 828                 errno = 0;
 829                 idx = strtoul(floating_veb_value, &end, 10);
 830                 if (errno || end == NULL)
 831                         return -1;
 832                 while (isblank(*end))
 833                         end++;
 834                 if (*end == '-') {
 835                         min = idx;
 836                 } else if ((*end == ';') || (*end == '\0')) {
 837                         max = idx;
 838                         if (min == I40E_MAX_VF)
 839                                 min = idx;
 840                         if (max >= I40E_MAX_VF)
 841                                 max = I40E_MAX_VF - 1;
 842                         for (idx = min; idx <= max; idx++) {
 843                                 vf_floating_veb[idx] = true;
 844                                 count++;
 845                         }
 846                         min = I40E_MAX_VF;
 847                 } else {
 848                         return -1;
 849                 }
 850                 floating_veb_value = end + 1;
 851         } while (*end != '\0');
 852
 853         if (count == 0)
 854                 return -1;
 855
 856         return 0;
 857 }
 858
 859 static void
 860 config_vf_floating_veb(struct rte_devargs *devargs,
 861                        uint16_t floating_veb,
 862                        bool *vf_floating_veb)
 863 {
 864         struct rte_kvargs *kvlist;
 865         int i;
 866         const char *floating_veb_list = ETH_I40E_FLOATING_VEB_LIST_ARG;
 867
 868         if (!floating_veb)
 869                 return;
 870         /* All the VFs attach to the floating VEB by default
 871          * when the floating VEB is enabled.
 872          */
 873         for (i = 0; i < I40E_MAX_VF; i++)
 874                 vf_floating_veb[i] = true;
 875
 876         if (devargs == NULL)
 877                 return;
 878
 879         kvlist = rte_kvargs_parse(devargs->args, NULL);
 880         if (kvlist == NULL)
 881                 return;
 882
 883         if (!rte_kvargs_count(kvlist, floating_veb_list)) {
 884                 rte_kvargs_free(kvlist);
 885                 return;
 886         }
 887         /* When the floating_veb_list parameter exists, all the VFs
 888          * will attach to the legacy VEB firstly, then configure VFs
 889          * to the floating VEB according to the floating_veb_list.
 890          */
 891         if (rte_kvargs_process(kvlist, floating_veb_list,
 892                                floating_veb_list_handler,
 893                                vf_floating_veb) < 0) {
 894                 rte_kvargs_free(kvlist);
 895                 return;
 896         }
 897         rte_kvargs_free(kvlist);
 898 }
 899
 900 static int
 901 i40e_check_floating_handler(__rte_unused const char *key,
 902                             const char *value,
 903                             __rte_unused void *opaque)
 904 {
 905         if (strcmp(value, "1"))
 906                 return -1;
 907
 908         return 0;
 909 }
 910
 911 static int
 912 is_floating_veb_supported(struct rte_devargs *devargs)
 913 {
 914         struct rte_kvargs *kvlist;
 915         const char *floating_veb_key = ETH_I40E_FLOATING_VEB_ARG;
 916
 917         if (devargs == NULL)
 918                 return 0;
 919
 920         kvlist = rte_kvargs_parse(devargs->args, NULL);
 921         if (kvlist == NULL)
 922                 return 0;
 923
 924         if (!rte_kvargs_count(kvlist, floating_veb_key)) {
 925                 rte_kvargs_free(kvlist);
 926                 return 0;
 927         }
 928         /* Floating VEB is enabled when there's key-value:
 929          * enable_floating_veb=1
 930          */
 931         if (rte_kvargs_process(kvlist, floating_veb_key,
 932                                i40e_check_floating_handler, NULL) < 0) {
 933                 rte_kvargs_free(kvlist);
 934                 return 0;
 935         }
 936         rte_kvargs_free(kvlist);
 937
 938         return 1;
 939 }
 940
 941 static void
 942 config_floating_veb(struct rte_eth_dev *dev)
 943 {
 944         struct rte_pci_device *pci_dev = dev->pci_dev;
 945         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 946         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 947
 948         memset(pf->floating_veb_list, 0, sizeof(pf->floating_veb_list));
 949
 950         if (hw->aq.fw_maj_ver >= FLOATING_VEB_SUPPORTED_FW_MAJ) {
 951                 pf->floating_veb =
 952                         is_floating_veb_supported(pci_dev->device.devargs);
 953                 config_vf_floating_veb(pci_dev->device.devargs,
 954                                        pf->floating_veb,
 955                                        pf->floating_veb_list);
 956         } else {
 957                 pf->floating_veb = false;
 958         }
 959 }
 960
 961 #define I40E_L2_TAGS_S_TAG_SHIFT 1
 962 #define I40E_L2_TAGS_S_TAG_MASK I40E_MASK(0x1, I40E_L2_TAGS_S_TAG_SHIFT)
 963
 964 #define ETH_I40E_SUPPORT_MULTI_DRIVER   "support-multi-driver"
 965 RTE_PMD_REGISTER_PARAM_STRING(net_i40e,
 966                               ETH_I40E_SUPPORT_MULTI_DRIVER "=0|1");
 967
 968 static int
 969 i40e_parse_multi_drv_handler(__rte_unused const char *key,
 970                               const char *value,
 971                               void *opaque)
 972 {
 973         struct i40e_pf *pf;
 974         unsigned long support_multi_driver;
 975         char *end;
 976
 977         pf = (struct i40e_pf *)opaque;
 978
 979         errno = 0;
 980         support_multi_driver = strtoul(value, &end, 10);
 981         if (errno != 0 || end == value || *end != 0) {
 982                 PMD_DRV_LOG(WARNING, "Wrong global configuration");
 983                 return -(EINVAL);
 984         }
 985
 986         if (support_multi_driver == 1 || support_multi_driver == 0)
 987                 pf->support_multi_driver = (bool)support_multi_driver;
 988         else
 989                 PMD_DRV_LOG(WARNING, "%s must be 1 or 0,",
 990                             "enable global configuration by default."
 991                             ETH_I40E_SUPPORT_MULTI_DRIVER);
 992         return 0;
 993 }
 994
 995 static int
 996 i40e_support_multi_driver(struct rte_eth_dev *dev)
 997 {
 998         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 999         struct rte_pci_device *pci_dev = dev->pci_dev;
1000         static const char *valid_keys[] = {
1001                 ETH_I40E_SUPPORT_MULTI_DRIVER, NULL};
1002         struct rte_kvargs *kvlist;
1003
1004         /* Enable global configuration by default */
1005         pf->support_multi_driver = false;
1006
1007         if (!pci_dev->device.devargs)
1008                 return 0;
1009
1010         kvlist = rte_kvargs_parse(pci_dev->device.devargs->args, valid_keys);
1011         if (!kvlist)
1012                 return -EINVAL;
1013
1014         if (rte_kvargs_count(kvlist, ETH_I40E_SUPPORT_MULTI_DRIVER) > 1)
1015                 PMD_DRV_LOG(WARNING, "More than one argument \"%s\" and only "
1016                             "the first invalid or last valid one is used !",
1017                             ETH_I40E_SUPPORT_MULTI_DRIVER);
1018
1019         if (rte_kvargs_process(kvlist, ETH_I40E_SUPPORT_MULTI_DRIVER,
1020                                i40e_parse_multi_drv_handler, pf) < 0) {
1021                 rte_kvargs_free(kvlist);
1022                 return -EINVAL;
1023         }
1024
1025         rte_kvargs_free(kvlist);
1026         return 0;
1027 }
1028
1029 static int
1030 eth_i40e_dev_init(struct rte_eth_dev *dev)
1031 {
1032         struct rte_pci_device *pci_dev;
1033         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1034         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1035         struct i40e_vsi *vsi;
1036         int ret;
1037         uint32_t len;
1038         uint8_t aq_fail = 0;
1039
1040         PMD_INIT_FUNC_TRACE();
1041
1042         dev->dev_ops = &i40e_eth_dev_ops;
1043         dev->rx_pkt_burst = i40e_recv_pkts;
1044         dev->tx_pkt_burst = i40e_xmit_pkts;
1045
1046         /* for secondary processes, we don't initialise any further as primary
1047          * has already done this work. Only check we don't need a different
1048          * RX function */
1049         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
1050                 i40e_set_rx_function(dev);
1051                 i40e_set_tx_function(dev);
1052                 return 0;
1053         }
1054         pci_dev = dev->pci_dev;
1055
1056         rte_eth_copy_pci_info(dev, pci_dev);
1057
1058         pf->adapter = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
1059         pf->adapter->eth_dev = dev;
1060         pf->dev_data = dev->data;
1061
1062         hw->back = I40E_PF_TO_ADAPTER(pf);
1063         hw->hw_addr = (uint8_t *)(pci_dev->mem_resource[0].addr);
1064         if (!hw->hw_addr) {
1065                 PMD_INIT_LOG(ERR, "Hardware is not available, "
1066                              "as address is NULL");
1067                 return -ENODEV;
1068         }
1069
1070         hw->vendor_id = pci_dev->id.vendor_id;
1071         hw->device_id = pci_dev->id.device_id;
1072         hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id;
1073         hw->subsystem_device_id = pci_dev->id.subsystem_device_id;
1074         hw->bus.device = pci_dev->addr.devid;
1075         hw->bus.func = pci_dev->addr.function;
1076         hw->adapter_stopped = 0;
1077
1078         /* Check if need to support multi-driver */
1079         i40e_support_multi_driver(dev);
1080
1081         /* Make sure all is clean before doing PF reset */
1082         i40e_clear_hw(hw);
1083
1084         /* Reset here to make sure all is clean for each PF */
1085         ret = i40e_pf_reset(hw);
1086         if (ret) {
1087                 PMD_INIT_LOG(ERR, "Failed to reset pf: %d", ret);
1088                 return ret;
1089         }
1090
1091         /* Initialize the shared code (base driver) */
1092         ret = i40e_init_shared_code(hw);
1093         if (ret) {
1094                 PMD_INIT_LOG(ERR, "Failed to init shared code (base driver): %d", ret);
1095                 return ret;
1096         }
1097
1098         /* Initialize the parameters for adminq */
1099         i40e_init_adminq_parameter(hw);
1100         ret = i40e_init_adminq(hw);
1101         if (ret != I40E_SUCCESS) {
1102                 PMD_INIT_LOG(ERR, "Failed to init adminq: %d", ret);
1103                 return -EIO;
1104         }
1105         PMD_INIT_LOG(INFO, "FW %d.%d API %d.%d NVM %02d.%02d.%02d eetrack %04x",
1106                      hw->aq.fw_maj_ver, hw->aq.fw_min_ver,
1107                      hw->aq.api_maj_ver, hw->aq.api_min_ver,
1108                      ((hw->nvm.version >> 12) & 0xf),
1109                      ((hw->nvm.version >> 4) & 0xff),
1110                      (hw->nvm.version & 0xf), hw->nvm.eetrack);
1111
1112         /* Initialize the hardware */
1113         i40e_hw_init(dev);
1114
1115         i40e_config_automask(pf);
1116
1117         /*
1118          * To work around the NVM issue, initialize registers
1119          * for flexible payload and packet type of QinQ by
1120          * software. It should be removed once issues are fixed
1121          * in NVM.
1122          */
1123         if (!pf->support_multi_driver)
1124                 i40e_GLQF_reg_init(hw);
1125
1126         /* Initialize the input set for filters (hash and fd) to default value */
1127         i40e_filter_input_set_init(pf);
1128
1129         /* Need the special FW version to support floating VEB */
1130         config_floating_veb(dev);
1131         /* Clear PXE mode */
1132         i40e_clear_pxe_mode(hw);
1133         ret = i40e_dev_sync_phy_type(hw);
1134         if (ret) {
1135                 PMD_INIT_LOG(ERR, "Failed to sync phy type: %d", ret);
1136                 goto err_sync_phy_type;
1137         }
1138         /*
1139          * On X710, performance number is far from the expectation on recent
1140          * firmware versions. The fix for this issue may not be integrated in
1141          * the following firmware version. So the workaround in software driver
1142          * is needed. It needs to modify the initial values of 3 internal only
1143          * registers. Note that the workaround can be removed when it is fixed
1144          * in firmware in the future.
1145          */
1146         i40e_configure_registers(hw);
1147
1148         /* Get hw capabilities */
1149         ret = i40e_get_cap(hw);
1150         if (ret != I40E_SUCCESS) {
1151                 PMD_INIT_LOG(ERR, "Failed to get capabilities: %d", ret);
1152                 goto err_get_capabilities;
1153         }
1154
1155         /* Initialize parameters for PF */
1156         ret = i40e_pf_parameter_init(dev);
1157         if (ret != 0) {
1158                 PMD_INIT_LOG(ERR, "Failed to do parameter init: %d", ret);
1159                 goto err_parameter_init;
1160         }
1161
1162         /* Initialize the queue management */
1163         ret = i40e_res_pool_init(&pf->qp_pool, 0, hw->func_caps.num_tx_qp);
1164         if (ret < 0) {
1165                 PMD_INIT_LOG(ERR, "Failed to init queue pool");
1166                 goto err_qp_pool_init;
1167         }
1168         ret = i40e_res_pool_init(&pf->msix_pool, 1,
1169                                 hw->func_caps.num_msix_vectors - 1);
1170         if (ret < 0) {
1171                 PMD_INIT_LOG(ERR, "Failed to init MSIX pool");
1172                 goto err_msix_pool_init;
1173         }
1174
1175         /* Initialize lan hmc */
1176         ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
1177                                 hw->func_caps.num_rx_qp, 0, 0);
1178         if (ret != I40E_SUCCESS) {
1179                 PMD_INIT_LOG(ERR, "Failed to init lan hmc: %d", ret);
1180                 goto err_init_lan_hmc;
1181         }
1182
1183         /* Configure lan hmc */
1184         ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
1185         if (ret != I40E_SUCCESS) {
1186                 PMD_INIT_LOG(ERR, "Failed to configure lan hmc: %d", ret);
1187                 goto err_configure_lan_hmc;
1188         }
1189
1190         /* Get and check the mac address */
1191         i40e_get_mac_addr(hw, hw->mac.addr);
1192         if (i40e_validate_mac_addr(hw->mac.addr) != I40E_SUCCESS) {
1193                 PMD_INIT_LOG(ERR, "mac address is not valid");
1194                 ret = -EIO;
1195                 goto err_get_mac_addr;
1196         }
1197         /* Copy the permanent MAC address */
1198         ether_addr_copy((struct ether_addr *) hw->mac.addr,
1199                         (struct ether_addr *) hw->mac.perm_addr);
1200
1201         /* Disable flow control */
1202         hw->fc.requested_mode = I40E_FC_NONE;
1203         i40e_set_fc(hw, &aq_fail, TRUE);
1204
1205         /* Set the global registers with default ether type value */
1206         if (!pf->support_multi_driver) {
1207                 ret = i40e_vlan_tpid_set(dev, ETH_VLAN_TYPE_OUTER,
1208                                          ETHER_TYPE_VLAN);
1209                 if (ret != I40E_SUCCESS) {
1210                         PMD_INIT_LOG(ERR, "Failed to set the default outer "
1211                                      "VLAN ether type");
1212                         goto err_setup_pf_switch;
1213                 }
1214         }
1215
1216         /* PF setup, which includes VSI setup */
1217         ret = i40e_pf_setup(pf);
1218         if (ret) {
1219                 PMD_INIT_LOG(ERR, "Failed to setup pf switch: %d", ret);
1220                 goto err_setup_pf_switch;
1221         }
1222
1223         /* reset all stats of the device, including pf and main vsi */
1224         i40e_dev_stats_reset(dev);
1225
1226         vsi = pf->main_vsi;
1227
1228         /* Disable double vlan by default */
1229         i40e_vsi_config_double_vlan(vsi, FALSE);
1230
1231         /* Disable S-TAG identification when floating_veb is disabled */
1232         if (!pf->floating_veb) {
1233                 ret = I40E_READ_REG(hw, I40E_PRT_L2TAGSEN);
1234                 if (ret & I40E_L2_TAGS_S_TAG_MASK) {
1235                         ret &= ~I40E_L2_TAGS_S_TAG_MASK;
1236                         I40E_WRITE_REG(hw, I40E_PRT_L2TAGSEN, ret);
1237                 }
1238         }
1239
1240         if (!vsi->max_macaddrs)
1241                 len = ETHER_ADDR_LEN;
1242         else
1243                 len = ETHER_ADDR_LEN * vsi->max_macaddrs;
1244
1245         /* Should be after VSI initialized */
1246         dev->data->mac_addrs = rte_zmalloc("i40e", len, 0);
1247         if (!dev->data->mac_addrs) {
1248                 PMD_INIT_LOG(ERR, "Failed to allocated memory "
1249                                         "for storing mac address");
1250                 goto err_mac_alloc;
1251         }
1252         ether_addr_copy((struct ether_addr *)hw->mac.perm_addr,
1253                                         &dev->data->mac_addrs[0]);
1254
1255         /* initialize pf host driver to setup SRIOV resource if applicable */
1256         i40e_pf_host_init(dev);
1257
1258         /* register callback func to eal lib */
1259         rte_intr_callback_register(&(pci_dev->intr_handle),
1260                 i40e_dev_interrupt_handler, (void *)dev);
1261
1262         /* configure and enable device interrupt */
1263         i40e_pf_config_irq0(hw, TRUE);
1264         i40e_pf_enable_irq0(hw);
1265
1266         /* enable uio intr after callback register */
1267         rte_intr_enable(&(pci_dev->intr_handle));
1268         /*
1269          * Add an ethertype filter to drop all flow control frames transmitted
1270          * from VSIs. By doing so, we stop VF from sending out PAUSE or PFC
1271          * frames to wire.
1272          */
1273         i40e_add_tx_flow_control_drop_filter(pf);
1274
1275         /* Set the max frame size to 0x2600 by default,
1276          * in case other drivers changed the default value.
1277          */
1278         i40e_aq_set_mac_config(hw, I40E_FRAME_SIZE_MAX, TRUE, 0, NULL);
1279
1280         /* initialize mirror rule list */
1281         TAILQ_INIT(&pf->mirror_list);
1282
1283         /* Init dcb to sw mode by default */
1284         ret = i40e_dcb_init_configure(dev, TRUE);
1285         if (ret != I40E_SUCCESS) {
1286                 PMD_INIT_LOG(INFO, "Failed to init dcb.");
1287                 pf->flags &= ~I40E_FLAG_DCB;
1288         }
1289
1290         return 0;
1291
1292 err_mac_alloc:
1293         i40e_vsi_release(pf->main_vsi);
1294 err_setup_pf_switch:
1295 err_get_mac_addr:
1296 err_configure_lan_hmc:
1297         (void)i40e_shutdown_lan_hmc(hw);
1298 err_init_lan_hmc:
1299         i40e_res_pool_destroy(&pf->msix_pool);
1300 err_msix_pool_init:
1301         i40e_res_pool_destroy(&pf->qp_pool);
1302 err_qp_pool_init:
1303 err_parameter_init:
1304 err_get_capabilities:
1305 err_sync_phy_type:
1306         (void)i40e_shutdown_adminq(hw);
1307
1308         return ret;
1309 }
1310
1311 static int
1312 eth_i40e_dev_uninit(struct rte_eth_dev *dev)
1313 {
1314         struct rte_pci_device *pci_dev;
1315         struct i40e_hw *hw;
1316         struct i40e_filter_control_settings settings;
1317         int ret;
1318         uint8_t aq_fail = 0;
1319         int retries = 0;
1320
1321         PMD_INIT_FUNC_TRACE();
1322
1323         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1324                 return 0;
1325
1326         hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1327         pci_dev = dev->pci_dev;
1328
1329         if (hw->adapter_stopped == 0)
1330                 i40e_dev_close(dev);
1331
1332         dev->dev_ops = NULL;
1333         dev->rx_pkt_burst = NULL;
1334         dev->tx_pkt_burst = NULL;
1335
1336         /* Clear PXE mode */
1337         i40e_clear_pxe_mode(hw);
1338
1339         /* Unconfigure filter control */
1340         memset(&settings, 0, sizeof(settings));
1341         ret = i40e_set_filter_control(hw, &settings);
1342         if (ret)
1343                 PMD_INIT_LOG(WARNING, "setup_pf_filter_control failed: %d",
1344                                         ret);
1345
1346         /* Disable flow control */
1347         hw->fc.requested_mode = I40E_FC_NONE;
1348         i40e_set_fc(hw, &aq_fail, TRUE);
1349
1350         /* uninitialize pf host driver */
1351         i40e_pf_host_uninit(dev);
1352
1353         rte_free(dev->data->mac_addrs);
1354         dev->data->mac_addrs = NULL;
1355
1356         /* disable uio intr before callback unregister */
1357         rte_intr_disable(&(pci_dev->intr_handle));
1358
1359         /* unregister callback func to eal lib */
1360         do {
1361                 ret = rte_intr_callback_unregister(&(pci_dev->intr_handle),
1362                                 i40e_dev_interrupt_handler, (void *)dev);
1363                 if (ret >= 0) {
1364                         break;
1365                 } else if (ret != -EAGAIN) {
1366                         PMD_INIT_LOG(ERR,
1367                                  "intr callback unregister failed: %d",
1368                                  ret);
1369                         return ret;
1370                 }
1371                 i40e_msec_delay(500);
1372         } while (retries++ < 5);
1373
1374         return 0;
1375 }
1376
1377 static int
1378 i40e_dev_configure(struct rte_eth_dev *dev)
1379 {
1380         struct i40e_adapter *ad =
1381                 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
1382         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1383         enum rte_eth_rx_mq_mode mq_mode = dev->data->dev_conf.rxmode.mq_mode;
1384         int i, ret;
1385
1386         /* Initialize to TRUE. If any of Rx queues doesn't meet the
1387          * bulk allocation or vector Rx preconditions we will reset it.
1388          */
1389         ad->rx_bulk_alloc_allowed = true;
1390         ad->rx_vec_allowed = true;
1391         ad->tx_simple_allowed = true;
1392         ad->tx_vec_allowed = true;
1393
1394         if (dev->data->dev_conf.fdir_conf.mode == RTE_FDIR_MODE_PERFECT) {
1395                 ret = i40e_fdir_setup(pf);
1396                 if (ret != I40E_SUCCESS) {
1397                         PMD_DRV_LOG(ERR, "Failed to setup flow director.");
1398                         return -ENOTSUP;
1399                 }
1400                 ret = i40e_fdir_configure(dev);
1401                 if (ret < 0) {
1402                         PMD_DRV_LOG(ERR, "failed to configure fdir.");
1403                         goto err;
1404                 }
1405         } else
1406                 i40e_fdir_teardown(pf);
1407
1408         ret = i40e_dev_init_vlan(dev);
1409         if (ret < 0)
1410                 goto err;
1411
1412         /* VMDQ setup.
1413          *  Needs to move VMDQ setting out of i40e_pf_config_mq_rx() as VMDQ and
1414          *  RSS setting have different requirements.
1415          *  General PMD driver call sequence are NIC init, configure,
1416          *  rx/tx_queue_setup and dev_start. In rx/tx_queue_setup() function, it
1417          *  will try to lookup the VSI that specific queue belongs to if VMDQ
1418          *  applicable. So, VMDQ setting has to be done before
1419          *  rx/tx_queue_setup(). This function is good  to place vmdq_setup.
1420          *  For RSS setting, it will try to calculate actual configured RX queue
1421          *  number, which will be available after rx_queue_setup(). dev_start()
1422          *  function is good to place RSS setup.
1423          */
1424         if (mq_mode & ETH_MQ_RX_VMDQ_FLAG) {
1425                 ret = i40e_vmdq_setup(dev);
1426                 if (ret)
1427                         goto err;
1428         }
1429
1430         if (mq_mode & ETH_MQ_RX_DCB_FLAG) {
1431                 ret = i40e_dcb_setup(dev);
1432                 if (ret) {
1433                         PMD_DRV_LOG(ERR, "failed to configure DCB.");
1434                         goto err_dcb;
1435                 }
1436         }
1437
1438         return 0;
1439
1440 err_dcb:
1441         /* need to release vmdq resource if exists */
1442         for (i = 0; i < pf->nb_cfg_vmdq_vsi; i++) {
1443                 i40e_vsi_release(pf->vmdq[i].vsi);
1444                 pf->vmdq[i].vsi = NULL;
1445         }
1446         rte_free(pf->vmdq);
1447         pf->vmdq = NULL;
1448 err:
1449         /* need to release fdir resource if exists */
1450         i40e_fdir_teardown(pf);
1451         return ret;
1452 }
1453
1454 void
1455 i40e_vsi_queues_unbind_intr(struct i40e_vsi *vsi)
1456 {
1457         struct rte_eth_dev *dev = vsi->adapter->eth_dev;
1458         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1459         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
1460         uint16_t msix_vect = vsi->msix_intr;
1461         uint16_t i;
1462
1463         for (i = 0; i < vsi->nb_qps; i++) {
1464                 I40E_WRITE_REG(hw, I40E_QINT_TQCTL(vsi->base_queue + i), 0);
1465                 I40E_WRITE_REG(hw, I40E_QINT_RQCTL(vsi->base_queue + i), 0);
1466                 rte_wmb();
1467         }
1468
1469         if (vsi->type != I40E_VSI_SRIOV) {
1470                 if (!rte_intr_allow_others(intr_handle)) {
1471                         I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0,
1472                                        I40E_PFINT_LNKLST0_FIRSTQ_INDX_MASK);
1473                         I40E_WRITE_REG(hw,
1474                                        I40E_PFINT_ITR0(I40E_ITR_INDEX_DEFAULT),
1475                                        0);
1476                 } else {
1477                         I40E_WRITE_REG(hw, I40E_PFINT_LNKLSTN(msix_vect - 1),
1478                                        I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK);
1479                         I40E_WRITE_REG(hw,
1480                                        I40E_PFINT_ITRN(I40E_ITR_INDEX_DEFAULT,
1481                                                        msix_vect - 1), 0);
1482                 }
1483         } else {
1484                 uint32_t reg;
1485                 reg = (hw->func_caps.num_msix_vectors_vf - 1) *
1486                         vsi->user_param + (msix_vect - 1);
1487
1488                 I40E_WRITE_REG(hw, I40E_VPINT_LNKLSTN(reg),
1489                                I40E_VPINT_LNKLSTN_FIRSTQ_INDX_MASK);
1490         }
1491         I40E_WRITE_FLUSH(hw);
1492 }
1493
1494 static void
1495 __vsi_queues_bind_intr(struct i40e_vsi *vsi, uint16_t msix_vect,
1496                        int base_queue, int nb_queue)
1497 {
1498         int i;
1499         uint32_t val;
1500         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
1501         struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
1502
1503         /* Bind all RX queues to allocated MSIX interrupt */
1504         for (i = 0; i < nb_queue; i++) {
1505                 val = (msix_vect << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
1506                         I40E_QINT_RQCTL_ITR_INDX_MASK |
1507                         ((base_queue + i + 1) <<
1508                          I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
1509                         (0 << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) |
1510                         I40E_QINT_RQCTL_CAUSE_ENA_MASK;
1511
1512                 if (i == nb_queue - 1)
1513                         val |= I40E_QINT_RQCTL_NEXTQ_INDX_MASK;
1514                 I40E_WRITE_REG(hw, I40E_QINT_RQCTL(base_queue + i), val);
1515         }
1516
1517         /* Write first RX queue to Link list register as the head element */
1518         if (vsi->type != I40E_VSI_SRIOV) {
1519                 uint16_t interval =
1520                         i40e_calc_itr_interval(RTE_LIBRTE_I40E_ITR_INTERVAL,
1521                                                pf->support_multi_driver);
1522
1523                 if (msix_vect == I40E_MISC_VEC_ID) {
1524                         I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0,
1525                                        (base_queue <<
1526                                         I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT) |
1527                                        (0x0 <<
1528                                         I40E_PFINT_LNKLST0_FIRSTQ_TYPE_SHIFT));
1529                         I40E_WRITE_REG(hw,
1530                                        I40E_PFINT_ITR0(I40E_ITR_INDEX_DEFAULT),
1531                                        interval);
1532                 } else {
1533                         I40E_WRITE_REG(hw, I40E_PFINT_LNKLSTN(msix_vect - 1),
1534                                        (base_queue <<
1535                                         I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) |
1536                                        (0x0 <<
1537                                         I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT));
1538                         I40E_WRITE_REG(hw,
1539                                        I40E_PFINT_ITRN(I40E_ITR_INDEX_DEFAULT,
1540                                                        msix_vect - 1),
1541                                        interval);
1542                 }
1543         } else {
1544                 uint32_t reg;
1545
1546                 if (msix_vect == I40E_MISC_VEC_ID) {
1547                         I40E_WRITE_REG(hw,
1548                                        I40E_VPINT_LNKLST0(vsi->user_param),
1549                                        (base_queue <<
1550                                         I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT) |
1551                                        (0x0 <<
1552                                         I40E_VPINT_LNKLST0_FIRSTQ_TYPE_SHIFT));
1553                 } else {
1554                         /* num_msix_vectors_vf needs to minus irq0 */
1555                         reg = (hw->func_caps.num_msix_vectors_vf - 1) *
1556                                 vsi->user_param + (msix_vect - 1);
1557
1558                         I40E_WRITE_REG(hw, I40E_VPINT_LNKLSTN(reg),
1559                                        (base_queue <<
1560                                         I40E_VPINT_LNKLSTN_FIRSTQ_INDX_SHIFT) |
1561                                        (0x0 <<
1562                                         I40E_VPINT_LNKLSTN_FIRSTQ_TYPE_SHIFT));
1563                 }
1564         }
1565
1566         I40E_WRITE_FLUSH(hw);
1567 }
1568
1569 void
1570 i40e_vsi_queues_bind_intr(struct i40e_vsi *vsi)
1571 {
1572         struct rte_eth_dev *dev = vsi->adapter->eth_dev;
1573         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1574         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
1575         uint16_t msix_vect = vsi->msix_intr;
1576         uint16_t nb_msix = RTE_MIN(vsi->nb_msix, intr_handle->nb_efd);
1577         uint16_t queue_idx = 0;
1578         int record = 0;
1579         int i;
1580
1581         for (i = 0; i < vsi->nb_qps; i++) {
1582                 I40E_WRITE_REG(hw, I40E_QINT_TQCTL(vsi->base_queue + i), 0);
1583                 I40E_WRITE_REG(hw, I40E_QINT_RQCTL(vsi->base_queue + i), 0);
1584         }
1585
1586         /* VF bind interrupt */
1587         if (vsi->type == I40E_VSI_SRIOV) {
1588                 __vsi_queues_bind_intr(vsi, msix_vect,
1589                                        vsi->base_queue, vsi->nb_qps);
1590                 return;
1591         }
1592
1593         /* PF & VMDq bind interrupt */
1594         if (rte_intr_dp_is_en(intr_handle)) {
1595                 if (vsi->type == I40E_VSI_MAIN) {
1596                         queue_idx = 0;
1597                         record = 1;
1598                 } else if (vsi->type == I40E_VSI_VMDQ2) {
1599                         struct i40e_vsi *main_vsi =
1600                                 I40E_DEV_PRIVATE_TO_MAIN_VSI(vsi->adapter);
1601                         queue_idx = vsi->base_queue - main_vsi->nb_qps;
1602                         record = 1;
1603                 }
1604         }
1605
1606         for (i = 0; i < vsi->nb_used_qps; i++) {
1607                 if (nb_msix <= 1) {
1608                         if (!rte_intr_allow_others(intr_handle))
1609                                 /* allow to share MISC_VEC_ID */
1610                                 msix_vect = I40E_MISC_VEC_ID;
1611
1612                         /* no enough msix_vect, map all to one */
1613                         __vsi_queues_bind_intr(vsi, msix_vect,
1614                                                vsi->base_queue + i,
1615                                                vsi->nb_used_qps - i);
1616                         for (; !!record && i < vsi->nb_used_qps; i++)
1617                                 intr_handle->intr_vec[queue_idx + i] =
1618                                         msix_vect;
1619                         break;
1620                 }
1621                 /* 1:1 queue/msix_vect mapping */
1622                 __vsi_queues_bind_intr(vsi, msix_vect,
1623                                        vsi->base_queue + i, 1);
1624                 if (!!record)
1625                         intr_handle->intr_vec[queue_idx + i] = msix_vect;
1626
1627                 msix_vect++;
1628                 nb_msix--;
1629         }
1630 }
1631
1632 static void
1633 i40e_vsi_enable_queues_intr(struct i40e_vsi *vsi)
1634 {
1635         struct rte_eth_dev *dev = vsi->adapter->eth_dev;
1636         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1637         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
1638         struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
1639         uint16_t msix_intr, i;
1640
1641         if (rte_intr_allow_others(intr_handle) && !pf->support_multi_driver)
1642                 for (i = 0; i < vsi->nb_msix; i++) {
1643                         msix_intr = vsi->msix_intr + i;
1644                         I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTLN(msix_intr - 1),
1645                                        I40E_PFINT_DYN_CTLN_INTENA_MASK |
1646                                        I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
1647                                        I40E_PFINT_DYN_CTLN_ITR_INDX_MASK);
1648                 }
1649         else
1650                 I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0,
1651                                I40E_PFINT_DYN_CTL0_INTENA_MASK |
1652                                I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
1653                                I40E_PFINT_DYN_CTL0_ITR_INDX_MASK);
1654
1655         I40E_WRITE_FLUSH(hw);
1656 }
1657
1658 static void
1659 i40e_vsi_disable_queues_intr(struct i40e_vsi *vsi)
1660 {
1661         struct rte_eth_dev *dev = vsi->adapter->eth_dev;
1662         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1663         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
1664         struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
1665         uint16_t msix_intr, i;
1666
1667         if (rte_intr_allow_others(intr_handle) && !pf->support_multi_driver)
1668                 for (i = 0; i < vsi->nb_msix; i++) {
1669                         msix_intr = vsi->msix_intr + i;
1670                         I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTLN(msix_intr - 1),
1671                                        I40E_PFINT_DYN_CTLN_ITR_INDX_MASK);
1672                 }
1673         else
1674                 I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0,
1675                                I40E_PFINT_DYN_CTL0_ITR_INDX_MASK);
1676
1677         I40E_WRITE_FLUSH(hw);
1678 }
1679
1680 static inline uint8_t
1681 i40e_parse_link_speeds(uint16_t link_speeds)
1682 {
1683         uint8_t link_speed = I40E_LINK_SPEED_UNKNOWN;
1684
1685         if (link_speeds & ETH_LINK_SPEED_40G)
1686                 link_speed |= I40E_LINK_SPEED_40GB;
1687         if (link_speeds & ETH_LINK_SPEED_25G)
1688                 link_speed |= I40E_LINK_SPEED_25GB;
1689         if (link_speeds & ETH_LINK_SPEED_20G)
1690                 link_speed |= I40E_LINK_SPEED_20GB;
1691         if (link_speeds & ETH_LINK_SPEED_10G)
1692                 link_speed |= I40E_LINK_SPEED_10GB;
1693         if (link_speeds & ETH_LINK_SPEED_1G)
1694                 link_speed |= I40E_LINK_SPEED_1GB;
1695         if (link_speeds & ETH_LINK_SPEED_100M)
1696                 link_speed |= I40E_LINK_SPEED_100MB;
1697
1698         return link_speed;
1699 }
1700
1701 static int
1702 i40e_phy_conf_link(struct i40e_hw *hw,
1703                    uint8_t abilities,
1704                    uint8_t force_speed,
1705                    bool is_up)
1706 {
1707         enum i40e_status_code status;
1708         struct i40e_aq_get_phy_abilities_resp phy_ab;
1709         struct i40e_aq_set_phy_config phy_conf;
1710         enum i40e_aq_phy_type cnt;
1711         uint8_t avail_speed;
1712         uint32_t phy_type_mask = 0;
1713
1714         const uint8_t mask = I40E_AQ_PHY_FLAG_PAUSE_TX |
1715                         I40E_AQ_PHY_FLAG_PAUSE_RX |
1716                         I40E_AQ_PHY_FLAG_PAUSE_RX |
1717                         I40E_AQ_PHY_FLAG_LOW_POWER;
1718         int ret = -ENOTSUP;
1719
1720         /* To get phy capabilities of available speeds. */
1721         status = i40e_aq_get_phy_capabilities(hw, false, true, &phy_ab,
1722                                               NULL);
1723         if (status) {
1724                 PMD_DRV_LOG(ERR, "Failed to get PHY capabilities: %d\n",
1725                                 status);
1726                 return ret;
1727         }
1728         avail_speed = phy_ab.link_speed;
1729
1730         /* To get the current phy config. */
1731         status = i40e_aq_get_phy_capabilities(hw, false, false, &phy_ab,
1732                                               NULL);
1733         if (status) {
1734                 PMD_DRV_LOG(ERR, "Failed to get the current PHY config: %d\n",
1735                                 status);
1736                 return ret;
1737         }
1738
1739         /* If link needs to go up and it is in autoneg mode the speed is OK,
1740          * no need to set up again.
1741          */
1742         if (is_up && phy_ab.phy_type != 0 &&
1743                      abilities & I40E_AQ_PHY_AN_ENABLED &&
1744                      phy_ab.link_speed != 0)
1745                 return I40E_SUCCESS;
1746
1747         memset(&phy_conf, 0, sizeof(phy_conf));
1748
1749         /* bits 0-2 use the values from get_phy_abilities_resp */
1750         abilities &= ~mask;
1751         abilities |= phy_ab.abilities & mask;
1752
1753         phy_conf.abilities = abilities;
1754
1755         /* If link needs to go up, but the force speed is not supported,
1756          * Warn users and config the default available speeds.
1757          */
1758         if (is_up && !(force_speed & avail_speed)) {
1759                 PMD_DRV_LOG(WARNING, "Invalid speed setting, set to default!\n");
1760                 phy_conf.link_speed = avail_speed;
1761         } else {
1762                 phy_conf.link_speed = is_up ? force_speed : avail_speed;
1763         }
1764
1765         /* PHY type mask needs to include each type except PHY type extension */
1766         for (cnt = I40E_PHY_TYPE_SGMII; cnt < I40E_PHY_TYPE_25GBASE_KR; cnt++)
1767                 phy_type_mask |= 1 << cnt;
1768
1769         /* use get_phy_abilities_resp value for the rest */
1770         phy_conf.phy_type = is_up ? cpu_to_le32(phy_type_mask) : 0;
1771         phy_conf.phy_type_ext = is_up ? (I40E_AQ_PHY_TYPE_EXT_25G_KR |
1772                 I40E_AQ_PHY_TYPE_EXT_25G_CR | I40E_AQ_PHY_TYPE_EXT_25G_SR |
1773                 I40E_AQ_PHY_TYPE_EXT_25G_LR) : 0;
1774         phy_conf.fec_config = phy_ab.mod_type_ext;
1775         phy_conf.eee_capability = phy_ab.eee_capability;
1776         phy_conf.eeer = phy_ab.eeer_val;
1777         phy_conf.low_power_ctrl = phy_ab.d3_lpan;
1778
1779         PMD_DRV_LOG(DEBUG, "\tCurrent: abilities %x, link_speed %x",
1780                     phy_ab.abilities, phy_ab.link_speed);
1781         PMD_DRV_LOG(DEBUG, "\tConfig:  abilities %x, link_speed %x",
1782                     phy_conf.abilities, phy_conf.link_speed);
1783
1784         status = i40e_aq_set_phy_config(hw, &phy_conf, NULL);
1785         if (status)
1786                 return ret;
1787
1788         return I40E_SUCCESS;
1789 }
1790
1791 static int
1792 i40e_apply_link_speed(struct rte_eth_dev *dev)
1793 {
1794         uint8_t speed;
1795         uint8_t abilities = 0;
1796         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1797         struct rte_eth_conf *conf = &dev->data->dev_conf;
1798
1799         if (conf->link_speeds == ETH_LINK_SPEED_AUTONEG) {
1800                 conf->link_speeds = ETH_LINK_SPEED_40G |
1801                                     ETH_LINK_SPEED_25G |
1802                                     ETH_LINK_SPEED_20G |
1803                                     ETH_LINK_SPEED_10G |
1804                                     ETH_LINK_SPEED_1G |
1805                                     ETH_LINK_SPEED_100M;
1806         }
1807         speed = i40e_parse_link_speeds(conf->link_speeds);
1808         abilities |= I40E_AQ_PHY_ENABLE_ATOMIC_LINK |
1809                      I40E_AQ_PHY_AN_ENABLED |
1810                      I40E_AQ_PHY_LINK_ENABLED;
1811
1812         return i40e_phy_conf_link(hw, abilities, speed, true);
1813 }
1814
1815 static int
1816 i40e_dev_start(struct rte_eth_dev *dev)
1817 {
1818         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1819         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1820         struct i40e_vsi *main_vsi = pf->main_vsi;
1821         int ret, i;
1822         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1823         uint32_t intr_vector = 0;
1824
1825         hw->adapter_stopped = 0;
1826
1827         if (dev->data->dev_conf.link_speeds & ETH_LINK_SPEED_FIXED) {
1828                 PMD_INIT_LOG(ERR, "Invalid link_speeds for port %hhu; autonegotiation disabled",
1829                              dev->data->port_id);
1830                 return -EINVAL;
1831         }
1832
1833         rte_intr_disable(intr_handle);
1834
1835         if ((rte_intr_cap_multiple(intr_handle) ||
1836              !RTE_ETH_DEV_SRIOV(dev).active) &&
1837             dev->data->dev_conf.intr_conf.rxq != 0) {
1838                 intr_vector = dev->data->nb_rx_queues;
1839                 if (rte_intr_efd_enable(intr_handle, intr_vector))
1840                         return -1;
1841         }
1842
1843         if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
1844                 intr_handle->intr_vec =
1845                         rte_zmalloc("intr_vec",
1846                                     dev->data->nb_rx_queues * sizeof(int),
1847                                     0);
1848                 if (!intr_handle->intr_vec) {
1849                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
1850                                      " intr_vec\n", dev->data->nb_rx_queues);
1851                         return -ENOMEM;
1852                 }
1853         }
1854
1855         /* Initialize VSI */
1856         ret = i40e_dev_rxtx_init(pf);
1857         if (ret != I40E_SUCCESS) {
1858                 PMD_DRV_LOG(ERR, "Failed to init rx/tx queues");
1859                 goto err_up;
1860         }
1861
1862         /* Map queues with MSIX interrupt */
1863         main_vsi->nb_used_qps = dev->data->nb_rx_queues -
1864                 pf->nb_cfg_vmdq_vsi * RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM;
1865         i40e_vsi_queues_bind_intr(main_vsi);
1866         i40e_vsi_enable_queues_intr(main_vsi);
1867
1868         /* Map VMDQ VSI queues with MSIX interrupt */
1869         for (i = 0; i < pf->nb_cfg_vmdq_vsi; i++) {
1870                 pf->vmdq[i].vsi->nb_used_qps = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM;
1871                 i40e_vsi_queues_bind_intr(pf->vmdq[i].vsi);
1872                 i40e_vsi_enable_queues_intr(pf->vmdq[i].vsi);
1873         }
1874
1875         /* enable FDIR MSIX interrupt */
1876         if (pf->fdir.fdir_vsi) {
1877                 i40e_vsi_queues_bind_intr(pf->fdir.fdir_vsi);
1878                 i40e_vsi_enable_queues_intr(pf->fdir.fdir_vsi);
1879         }
1880
1881         /* Enable all queues which have been configured */
1882         ret = i40e_dev_switch_queues(pf, TRUE);
1883         if (ret != I40E_SUCCESS) {
1884                 PMD_DRV_LOG(ERR, "Failed to enable VSI");
1885                 goto err_up;
1886         }
1887
1888         /* Enable receiving broadcast packets */
1889         ret = i40e_aq_set_vsi_broadcast(hw, main_vsi->seid, true, NULL);
1890         if (ret != I40E_SUCCESS)
1891                 PMD_DRV_LOG(INFO, "fail to set vsi broadcast");
1892
1893         for (i = 0; i < pf->nb_cfg_vmdq_vsi; i++) {
1894                 ret = i40e_aq_set_vsi_broadcast(hw, pf->vmdq[i].vsi->seid,
1895                                                 true, NULL);
1896                 if (ret != I40E_SUCCESS)
1897                         PMD_DRV_LOG(INFO, "fail to set vsi broadcast");
1898         }
1899
1900         /* Apply link configure */
1901         ret = i40e_apply_link_speed(dev);
1902         if (I40E_SUCCESS != ret) {
1903                 PMD_DRV_LOG(ERR, "Fail to apply link setting");
1904                 goto err_up;
1905         }
1906
1907         if (!rte_intr_allow_others(intr_handle)) {
1908                 rte_intr_callback_unregister(intr_handle,
1909                                              i40e_dev_interrupt_handler,
1910                                              (void *)dev);
1911                 /* configure and enable device interrupt */
1912                 i40e_pf_config_irq0(hw, FALSE);
1913                 i40e_pf_enable_irq0(hw);
1914
1915                 if (dev->data->dev_conf.intr_conf.lsc != 0)
1916                         PMD_INIT_LOG(INFO, "lsc won't enable because of"
1917                                      " no intr multiplex\n");
1918         } else if (dev->data->dev_conf.intr_conf.lsc != 0) {
1919                 ret = i40e_aq_set_phy_int_mask(hw,
1920                                                ~(I40E_AQ_EVENT_LINK_UPDOWN |
1921                                                I40E_AQ_EVENT_MODULE_QUAL_FAIL |
1922                                                I40E_AQ_EVENT_MEDIA_NA), NULL);
1923                 if (ret != I40E_SUCCESS)
1924                         PMD_DRV_LOG(WARNING, "Fail to set phy mask");
1925
1926                 /* Call get_link_info aq commond to enable LSE */
1927                 i40e_dev_link_update(dev, 0);
1928         }
1929
1930         /* enable uio intr after callback register */
1931         rte_intr_enable(intr_handle);
1932
1933         return I40E_SUCCESS;
1934
1935 err_up:
1936         i40e_dev_switch_queues(pf, FALSE);
1937         i40e_dev_clear_queues(dev);
1938
1939         return ret;
1940 }
1941
1942 static void
1943 i40e_dev_stop(struct rte_eth_dev *dev)
1944 {
1945         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1946         struct i40e_vsi *main_vsi = pf->main_vsi;
1947         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1948         int i;
1949
1950         /* Disable all queues */
1951         i40e_dev_switch_queues(pf, FALSE);
1952
1953         /* un-map queues with interrupt registers */
1954         i40e_vsi_disable_queues_intr(main_vsi);
1955         i40e_vsi_queues_unbind_intr(main_vsi);
1956
1957         for (i = 0; i < pf->nb_cfg_vmdq_vsi; i++) {
1958                 i40e_vsi_disable_queues_intr(pf->vmdq[i].vsi);
1959                 i40e_vsi_queues_unbind_intr(pf->vmdq[i].vsi);
1960         }
1961
1962         if (pf->fdir.fdir_vsi) {
1963                 i40e_vsi_queues_unbind_intr(pf->fdir.fdir_vsi);
1964                 i40e_vsi_disable_queues_intr(pf->fdir.fdir_vsi);
1965         }
1966         /* Clear all queues and release memory */
1967         i40e_dev_clear_queues(dev);
1968
1969         /* Set link down */
1970         i40e_dev_set_link_down(dev);
1971
1972         if (!rte_intr_allow_others(intr_handle))
1973                 /* resume to the default handler */
1974                 rte_intr_callback_register(intr_handle,
1975                                            i40e_dev_interrupt_handler,
1976                                            (void *)dev);
1977
1978         /* Clean datapath event and queue/vec mapping */
1979         rte_intr_efd_disable(intr_handle);
1980         if (intr_handle->intr_vec) {
1981                 rte_free(intr_handle->intr_vec);
1982                 intr_handle->intr_vec = NULL;
1983         }
1984 }
1985
1986 static void
1987 i40e_dev_close(struct rte_eth_dev *dev)
1988 {
1989         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1990         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1991         struct i40e_mirror_rule *p_mirror;
1992         uint32_t reg;
1993         int i;
1994         int ret;
1995
1996         PMD_INIT_FUNC_TRACE();
1997
1998         i40e_dev_stop(dev);
1999         hw->adapter_stopped = 1;
2000
2001         /* Remove all mirror rules */
2002         while ((p_mirror = TAILQ_FIRST(&pf->mirror_list))) {
2003                 ret = i40e_aq_del_mirror_rule(hw,
2004                                               pf->main_vsi->veb->seid,
2005                                               p_mirror->rule_type,
2006                                               p_mirror->entries,
2007                                               p_mirror->num_entries,
2008                                               p_mirror->id);
2009                 if (ret < 0)
2010                         PMD_DRV_LOG(ERR, "failed to remove mirror rule: "
2011                                     "status = %d, aq_err = %d.", ret,
2012                                     hw->aq.asq_last_status);
2013
2014                 /* remove mirror software resource anyway */
2015                 TAILQ_REMOVE(&pf->mirror_list, p_mirror, rules);
2016                 rte_free(p_mirror);
2017                 pf->nb_mirror_rule--;
2018         }
2019
2020         i40e_dev_free_queues(dev);
2021
2022         /* Disable interrupt */
2023         i40e_pf_disable_irq0(hw);
2024         rte_intr_disable(&(dev->pci_dev->intr_handle));
2025
2026         i40e_fdir_teardown(pf);
2027
2028         /* shutdown and destroy the HMC */
2029         i40e_shutdown_lan_hmc(hw);
2030
2031         for (i = 0; i < pf->nb_cfg_vmdq_vsi; i++) {
2032                 i40e_vsi_release(pf->vmdq[i].vsi);
2033                 pf->vmdq[i].vsi = NULL;
2034         }
2035         rte_free(pf->vmdq);
2036         pf->vmdq = NULL;
2037
2038         /* release all the existing VSIs and VEBs */
2039         i40e_vsi_release(pf->main_vsi);
2040
2041         /* shutdown the adminq */
2042         i40e_aq_queue_shutdown(hw, true);
2043         i40e_shutdown_adminq(hw);
2044
2045         i40e_res_pool_destroy(&pf->qp_pool);
2046         i40e_res_pool_destroy(&pf->msix_pool);
2047
2048         /* force a PF reset to clean anything leftover */
2049         reg = I40E_READ_REG(hw, I40E_PFGEN_CTRL);
2050         I40E_WRITE_REG(hw, I40E_PFGEN_CTRL,
2051                         (reg | I40E_PFGEN_CTRL_PFSWR_MASK));
2052         I40E_WRITE_FLUSH(hw);
2053 }
2054
2055 static void
2056 i40e_dev_promiscuous_enable(struct rte_eth_dev *dev)
2057 {
2058         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2059         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2060         struct i40e_vsi *vsi = pf->main_vsi;
2061         int status;
2062
2063         status = i40e_aq_set_vsi_unicast_promiscuous(hw, vsi->seid,
2064                                                      true, NULL, true);
2065         if (status != I40E_SUCCESS)
2066                 PMD_DRV_LOG(ERR, "Failed to enable unicast promiscuous");
2067
2068         status = i40e_aq_set_vsi_multicast_promiscuous(hw, vsi->seid,
2069                                                         TRUE, NULL);
2070         if (status != I40E_SUCCESS)
2071                 PMD_DRV_LOG(ERR, "Failed to enable multicast promiscuous");
2072
2073 }
2074
2075 static void
2076 i40e_dev_promiscuous_disable(struct rte_eth_dev *dev)
2077 {
2078         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2079         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2080         struct i40e_vsi *vsi = pf->main_vsi;
2081         int status;
2082
2083         status = i40e_aq_set_vsi_unicast_promiscuous(hw, vsi->seid,
2084                                                      false, NULL, true);
2085         if (status != I40E_SUCCESS)
2086                 PMD_DRV_LOG(ERR, "Failed to disable unicast promiscuous");
2087
2088         /* must remain in all_multicast mode */
2089         if (dev->data->all_multicast == 1)
2090                 return;
2091
2092         status = i40e_aq_set_vsi_multicast_promiscuous(hw, vsi->seid,
2093                                                         false, NULL);
2094         if (status != I40E_SUCCESS)
2095                 PMD_DRV_LOG(ERR, "Failed to disable multicast promiscuous");
2096 }
2097
2098 static void
2099 i40e_dev_allmulticast_enable(struct rte_eth_dev *dev)
2100 {
2101         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2102         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2103         struct i40e_vsi *vsi = pf->main_vsi;
2104         int ret;
2105
2106         ret = i40e_aq_set_vsi_multicast_promiscuous(hw, vsi->seid, TRUE, NULL);
2107         if (ret != I40E_SUCCESS)
2108                 PMD_DRV_LOG(ERR, "Failed to enable multicast promiscuous");
2109 }
2110
2111 static void
2112 i40e_dev_allmulticast_disable(struct rte_eth_dev *dev)
2113 {
2114         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2115         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2116         struct i40e_vsi *vsi = pf->main_vsi;
2117         int ret;
2118
2119         if (dev->data->promiscuous == 1)
2120                 return; /* must remain in all_multicast mode */
2121
2122         ret = i40e_aq_set_vsi_multicast_promiscuous(hw,
2123                                 vsi->seid, FALSE, NULL);
2124         if (ret != I40E_SUCCESS)
2125                 PMD_DRV_LOG(ERR, "Failed to disable multicast promiscuous");
2126 }
2127
2128 /*
2129  * Set device link up.
2130  */
2131 static int
2132 i40e_dev_set_link_up(struct rte_eth_dev *dev)
2133 {
2134         /* re-apply link speed setting */
2135         return i40e_apply_link_speed(dev);
2136 }
2137
2138 /*
2139  * Set device link down.
2140  */
2141 static int
2142 i40e_dev_set_link_down(struct rte_eth_dev *dev)
2143 {
2144         uint8_t speed = I40E_LINK_SPEED_UNKNOWN;
2145         uint8_t abilities = 0;
2146         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2147
2148         abilities = I40E_AQ_PHY_ENABLE_ATOMIC_LINK;
2149         return i40e_phy_conf_link(hw, abilities, speed, false);
2150 }
2151
2152 static inline void __attribute__((always_inline))
2153 update_link_no_wait(struct i40e_hw *hw, struct rte_eth_link *link)
2154 {
2155 /* Link status registers and values*/
2156 #define I40E_PRTMAC_LINKSTA             0x001E2420
2157 #define I40E_REG_LINK_UP                0x40000080
2158 #define I40E_PRTMAC_MACC                0x001E24E0
2159 #define I40E_REG_MACC_25GB              0x00020000
2160 #define I40E_REG_SPEED_MASK             0x38000000
2161 #define I40E_REG_SPEED_100MB            0x00000000
2162 #define I40E_REG_SPEED_1GB              0x08000000
2163 #define I40E_REG_SPEED_10GB             0x10000000
2164 #define I40E_REG_SPEED_20GB             0x20000000
2165 #define I40E_REG_SPEED_25_40GB          0x18000000
2166         uint32_t link_speed;
2167         uint32_t reg_val;
2168
2169         reg_val = I40E_READ_REG(hw, I40E_PRTMAC_LINKSTA);
2170         link_speed = reg_val & I40E_REG_SPEED_MASK;
2171         reg_val &= I40E_REG_LINK_UP;
2172         link->link_status = (reg_val == I40E_REG_LINK_UP) ? 1 : 0;
2173
2174         if (unlikely(link->link_status == 0))
2175                 return;
2176
2177         /* Parse the link status */
2178         switch (link_speed) {
2179         case I40E_REG_SPEED_100MB:
2180                 link->link_speed = ETH_SPEED_NUM_100M;
2181                 break;
2182         case I40E_REG_SPEED_1GB:
2183                 link->link_speed = ETH_SPEED_NUM_1G;
2184                 break;
2185         case I40E_REG_SPEED_10GB:
2186                 link->link_speed = ETH_SPEED_NUM_10G;
2187                 break;
2188         case I40E_REG_SPEED_20GB:
2189                 link->link_speed = ETH_SPEED_NUM_20G;
2190                 break;
2191         case I40E_REG_SPEED_25_40GB:
2192                 reg_val = I40E_READ_REG(hw, I40E_PRTMAC_MACC);
2193
2194                 if (reg_val & I40E_REG_MACC_25GB)
2195                         link->link_speed = ETH_SPEED_NUM_25G;
2196                 else
2197                         link->link_speed = ETH_SPEED_NUM_40G;
2198
2199                 break;
2200         default:
2201                 PMD_DRV_LOG(ERR, "Unknown link speed info %u", link_speed);
2202                 break;
2203         }
2204 }
2205
2206 static inline void __attribute__((always_inline))
2207 update_link_wait(struct i40e_hw *hw, struct rte_eth_link *link,
2208         bool enable_lse)
2209 {
2210 #define CHECK_INTERVAL             100  /* 100ms */
2211 #define MAX_REPEAT_TIME            10  /* 1s (10 * 100ms) in total */
2212         uint32_t rep_cnt = MAX_REPEAT_TIME;
2213         struct i40e_link_status link_status;
2214         int status;
2215
2216         memset(&link_status, 0, sizeof(link_status));
2217
2218         do {
2219                 /* Get link status information from hardware */
2220                 status = i40e_aq_get_link_info(hw, enable_lse,
2221                                                 &link_status, NULL);
2222                 if (unlikely(status != I40E_SUCCESS)) {
2223                         link->link_speed = ETH_SPEED_NUM_100M;
2224                         link->link_duplex = ETH_LINK_FULL_DUPLEX;
2225                         PMD_DRV_LOG(ERR, "Failed to get link info");
2226                         return;
2227                 }
2228
2229                 link->link_status = link_status.link_info & I40E_AQ_LINK_UP;
2230                 if (unlikely(link->link_status != 0))
2231                         break;
2232
2233                 rte_delay_ms(CHECK_INTERVAL);
2234         } while (--rep_cnt);
2235
2236         /* Parse the link status */
2237         switch (link_status.link_speed) {
2238         case I40E_LINK_SPEED_100MB:
2239                 link->link_speed = ETH_SPEED_NUM_100M;
2240                 break;
2241         case I40E_LINK_SPEED_1GB:
2242                 link->link_speed = ETH_SPEED_NUM_1G;
2243                 break;
2244         case I40E_LINK_SPEED_10GB:
2245                 link->link_speed = ETH_SPEED_NUM_10G;
2246                 break;
2247         case I40E_LINK_SPEED_20GB:
2248                 link->link_speed = ETH_SPEED_NUM_20G;
2249                 break;
2250         case I40E_LINK_SPEED_25GB:
2251                 link->link_speed = ETH_SPEED_NUM_25G;
2252                 break;
2253         case I40E_LINK_SPEED_40GB:
2254                 link->link_speed = ETH_SPEED_NUM_40G;
2255                 break;
2256         default:
2257                 link->link_speed = ETH_SPEED_NUM_100M;
2258                 break;
2259         }
2260 }
2261
2262 int
2263 i40e_dev_link_update(struct rte_eth_dev *dev,
2264                      int wait_to_complete)
2265 {
2266         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2267         struct rte_eth_link link, old;
2268         bool enable_lse = dev->data->dev_conf.intr_conf.lsc ? true : false;
2269
2270         memset(&link, 0, sizeof(link));
2271         memset(&old, 0, sizeof(old));
2272
2273         rte_i40e_dev_atomic_read_link_status(dev, &old);
2274
2275         /* i40e uses full duplex only */
2276         link.link_duplex = ETH_LINK_FULL_DUPLEX;
2277         link.link_autoneg = !(dev->data->dev_conf.link_speeds &
2278                         ETH_LINK_SPEED_FIXED);
2279
2280         if (!wait_to_complete)
2281                 update_link_no_wait(hw, &link);
2282         else
2283                 update_link_wait(hw, &link, enable_lse);
2284
2285         rte_i40e_dev_atomic_write_link_status(dev, &link);
2286         if (link.link_status == old.link_status)
2287                 return -1;
2288
2289         i40e_notify_all_vfs_link_status(dev);
2290
2291         return 0;
2292 }
2293
2294 /* Get all the statistics of a VSI */
2295 void
2296 i40e_update_vsi_stats(struct i40e_vsi *vsi)
2297 {
2298         struct i40e_eth_stats *oes = &vsi->eth_stats_offset;
2299         struct i40e_eth_stats *nes = &vsi->eth_stats;
2300         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2301         int idx = rte_le_to_cpu_16(vsi->info.stat_counter_idx);
2302
2303         i40e_stat_update_48(hw, I40E_GLV_GORCH(idx), I40E_GLV_GORCL(idx),
2304                             vsi->offset_loaded, &oes->rx_bytes,
2305                             &nes->rx_bytes);
2306         i40e_stat_update_48(hw, I40E_GLV_UPRCH(idx), I40E_GLV_UPRCL(idx),
2307                             vsi->offset_loaded, &oes->rx_unicast,
2308                             &nes->rx_unicast);
2309         i40e_stat_update_48(hw, I40E_GLV_MPRCH(idx), I40E_GLV_MPRCL(idx),
2310                             vsi->offset_loaded, &oes->rx_multicast,
2311                             &nes->rx_multicast);
2312         i40e_stat_update_48(hw, I40E_GLV_BPRCH(idx), I40E_GLV_BPRCL(idx),
2313                             vsi->offset_loaded, &oes->rx_broadcast,
2314                             &nes->rx_broadcast);
2315         /* exclude CRC bytes */
2316         nes->rx_bytes -= (nes->rx_unicast + nes->rx_multicast +
2317                 nes->rx_broadcast) * ETHER_CRC_LEN;
2318
2319         i40e_stat_update_32(hw, I40E_GLV_RDPC(idx), vsi->offset_loaded,
2320                             &oes->rx_discards, &nes->rx_discards);
2321         /* GLV_REPC not supported */
2322         /* GLV_RMPC not supported */
2323         i40e_stat_update_32(hw, I40E_GLV_RUPP(idx), vsi->offset_loaded,
2324                             &oes->rx_unknown_protocol,
2325                             &nes->rx_unknown_protocol);
2326         i40e_stat_update_48(hw, I40E_GLV_GOTCH(idx), I40E_GLV_GOTCL(idx),
2327                             vsi->offset_loaded, &oes->tx_bytes,
2328                             &nes->tx_bytes);
2329         i40e_stat_update_48(hw, I40E_GLV_UPTCH(idx), I40E_GLV_UPTCL(idx),
2330                             vsi->offset_loaded, &oes->tx_unicast,
2331                             &nes->tx_unicast);
2332         i40e_stat_update_48(hw, I40E_GLV_MPTCH(idx), I40E_GLV_MPTCL(idx),
2333                             vsi->offset_loaded, &oes->tx_multicast,
2334                             &nes->tx_multicast);
2335         i40e_stat_update_48(hw, I40E_GLV_BPTCH(idx), I40E_GLV_BPTCL(idx),
2336                             vsi->offset_loaded,  &oes->tx_broadcast,
2337                             &nes->tx_broadcast);
2338         /* exclude CRC bytes */
2339         nes->tx_bytes -= (nes->tx_unicast + nes->tx_multicast +
2340                 nes->tx_broadcast) * ETHER_CRC_LEN;
2341         /* GLV_TDPC not supported */
2342         i40e_stat_update_32(hw, I40E_GLV_TEPC(idx), vsi->offset_loaded,
2343                             &oes->tx_errors, &nes->tx_errors);
2344         vsi->offset_loaded = true;
2345
2346         PMD_DRV_LOG(DEBUG, "***************** VSI[%u] stats start *******************",
2347                     vsi->vsi_id);
2348         PMD_DRV_LOG(DEBUG, "rx_bytes:            %"PRIu64"", nes->rx_bytes);
2349         PMD_DRV_LOG(DEBUG, "rx_unicast:          %"PRIu64"", nes->rx_unicast);
2350         PMD_DRV_LOG(DEBUG, "rx_multicast:        %"PRIu64"", nes->rx_multicast);
2351         PMD_DRV_LOG(DEBUG, "rx_broadcast:        %"PRIu64"", nes->rx_broadcast);
2352         PMD_DRV_LOG(DEBUG, "rx_discards:         %"PRIu64"", nes->rx_discards);
2353         PMD_DRV_LOG(DEBUG, "rx_unknown_protocol: %"PRIu64"",
2354                     nes->rx_unknown_protocol);
2355         PMD_DRV_LOG(DEBUG, "tx_bytes:            %"PRIu64"", nes->tx_bytes);
2356         PMD_DRV_LOG(DEBUG, "tx_unicast:          %"PRIu64"", nes->tx_unicast);
2357         PMD_DRV_LOG(DEBUG, "tx_multicast:        %"PRIu64"", nes->tx_multicast);
2358         PMD_DRV_LOG(DEBUG, "tx_broadcast:        %"PRIu64"", nes->tx_broadcast);
2359         PMD_DRV_LOG(DEBUG, "tx_discards:         %"PRIu64"", nes->tx_discards);
2360         PMD_DRV_LOG(DEBUG, "tx_errors:           %"PRIu64"", nes->tx_errors);
2361         PMD_DRV_LOG(DEBUG, "***************** VSI[%u] stats end *******************",
2362                     vsi->vsi_id);
2363 }
2364
2365 static void
2366 i40e_read_stats_registers(struct i40e_pf *pf, struct i40e_hw *hw)
2367 {
2368         unsigned int i;
2369         struct i40e_hw_port_stats *ns = &pf->stats; /* new stats */
2370         struct i40e_hw_port_stats *os = &pf->stats_offset; /* old stats */
2371
2372         /* Get rx/tx bytes of internal transfer packets */
2373         i40e_stat_update_48(hw, I40E_GLV_GORCH(hw->port),
2374                         I40E_GLV_GORCL(hw->port),
2375                         pf->offset_loaded,
2376                         &pf->internal_rx_bytes_offset,
2377                         &pf->internal_rx_bytes);
2378
2379         i40e_stat_update_48(hw, I40E_GLV_GOTCH(hw->port),
2380                         I40E_GLV_GOTCL(hw->port),
2381                         pf->offset_loaded,
2382                         &pf->internal_tx_bytes_offset,
2383                         &pf->internal_tx_bytes);
2384
2385         /* Get statistics of struct i40e_eth_stats */
2386         i40e_stat_update_48(hw, I40E_GLPRT_GORCH(hw->port),
2387                             I40E_GLPRT_GORCL(hw->port),
2388                             pf->offset_loaded, &os->eth.rx_bytes,
2389                             &ns->eth.rx_bytes);
2390         i40e_stat_update_48(hw, I40E_GLPRT_UPRCH(hw->port),
2391                             I40E_GLPRT_UPRCL(hw->port),
2392                             pf->offset_loaded, &os->eth.rx_unicast,
2393                             &ns->eth.rx_unicast);
2394         i40e_stat_update_48(hw, I40E_GLPRT_MPRCH(hw->port),
2395                             I40E_GLPRT_MPRCL(hw->port),
2396                             pf->offset_loaded, &os->eth.rx_multicast,
2397                             &ns->eth.rx_multicast);
2398         i40e_stat_update_48(hw, I40E_GLPRT_BPRCH(hw->port),
2399                             I40E_GLPRT_BPRCL(hw->port),
2400                             pf->offset_loaded, &os->eth.rx_broadcast,
2401                             &ns->eth.rx_broadcast);
2402         /* Workaround: CRC size should not be included in byte statistics,
2403          * so subtract ETHER_CRC_LEN from the byte counter for each rx packet.
2404          */
2405         ns->eth.rx_bytes -= (ns->eth.rx_unicast + ns->eth.rx_multicast +
2406                 ns->eth.rx_broadcast) * ETHER_CRC_LEN + pf->internal_rx_bytes;
2407
2408         i40e_stat_update_32(hw, I40E_GLPRT_RDPC(hw->port),
2409                             pf->offset_loaded, &os->eth.rx_discards,
2410                             &ns->eth.rx_discards);
2411         /* GLPRT_REPC not supported */
2412         /* GLPRT_RMPC not supported */
2413         i40e_stat_update_32(hw, I40E_GLPRT_RUPP(hw->port),
2414                             pf->offset_loaded,
2415                             &os->eth.rx_unknown_protocol,
2416                             &ns->eth.rx_unknown_protocol);
2417         i40e_stat_update_48(hw, I40E_GLPRT_GOTCH(hw->port),
2418                             I40E_GLPRT_GOTCL(hw->port),
2419                             pf->offset_loaded, &os->eth.tx_bytes,
2420                             &ns->eth.tx_bytes);
2421         i40e_stat_update_48(hw, I40E_GLPRT_UPTCH(hw->port),
2422                             I40E_GLPRT_UPTCL(hw->port),
2423                             pf->offset_loaded, &os->eth.tx_unicast,
2424                             &ns->eth.tx_unicast);
2425         i40e_stat_update_48(hw, I40E_GLPRT_MPTCH(hw->port),
2426                             I40E_GLPRT_MPTCL(hw->port),
2427                             pf->offset_loaded, &os->eth.tx_multicast,
2428                             &ns->eth.tx_multicast);
2429         i40e_stat_update_48(hw, I40E_GLPRT_BPTCH(hw->port),
2430                             I40E_GLPRT_BPTCL(hw->port),
2431                             pf->offset_loaded, &os->eth.tx_broadcast,
2432                             &ns->eth.tx_broadcast);
2433         ns->eth.tx_bytes -= (ns->eth.tx_unicast + ns->eth.tx_multicast +
2434                 ns->eth.tx_broadcast) * ETHER_CRC_LEN + pf->internal_tx_bytes;
2435         /* GLPRT_TEPC not supported */
2436
2437         /* additional port specific stats */
2438         i40e_stat_update_32(hw, I40E_GLPRT_TDOLD(hw->port),
2439                             pf->offset_loaded, &os->tx_dropped_link_down,
2440                             &ns->tx_dropped_link_down);
2441         i40e_stat_update_32(hw, I40E_GLPRT_CRCERRS(hw->port),
2442                             pf->offset_loaded, &os->crc_errors,
2443                             &ns->crc_errors);
2444         i40e_stat_update_32(hw, I40E_GLPRT_ILLERRC(hw->port),
2445                             pf->offset_loaded, &os->illegal_bytes,
2446                             &ns->illegal_bytes);
2447         /* GLPRT_ERRBC not supported */
2448         i40e_stat_update_32(hw, I40E_GLPRT_MLFC(hw->port),
2449                             pf->offset_loaded, &os->mac_local_faults,
2450                             &ns->mac_local_faults);
2451         i40e_stat_update_32(hw, I40E_GLPRT_MRFC(hw->port),
2452                             pf->offset_loaded, &os->mac_remote_faults,
2453                             &ns->mac_remote_faults);
2454         i40e_stat_update_32(hw, I40E_GLPRT_RLEC(hw->port),
2455                             pf->offset_loaded, &os->rx_length_errors,
2456                             &ns->rx_length_errors);
2457         i40e_stat_update_32(hw, I40E_GLPRT_LXONRXC(hw->port),
2458                             pf->offset_loaded, &os->link_xon_rx,
2459                             &ns->link_xon_rx);
2460         i40e_stat_update_32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
2461                             pf->offset_loaded, &os->link_xoff_rx,
2462                             &ns->link_xoff_rx);
2463         for (i = 0; i < 8; i++) {
2464                 i40e_stat_update_32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
2465                                     pf->offset_loaded,
2466                                     &os->priority_xon_rx[i],
2467                                     &ns->priority_xon_rx[i]);
2468                 i40e_stat_update_32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
2469                                     pf->offset_loaded,
2470                                     &os->priority_xoff_rx[i],
2471                                     &ns->priority_xoff_rx[i]);
2472         }
2473         i40e_stat_update_32(hw, I40E_GLPRT_LXONTXC(hw->port),
2474                             pf->offset_loaded, &os->link_xon_tx,
2475                             &ns->link_xon_tx);
2476         i40e_stat_update_32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
2477                             pf->offset_loaded, &os->link_xoff_tx,
2478                             &ns->link_xoff_tx);
2479         for (i = 0; i < 8; i++) {
2480                 i40e_stat_update_32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
2481                                     pf->offset_loaded,
2482                                     &os->priority_xon_tx[i],
2483                                     &ns->priority_xon_tx[i]);
2484                 i40e_stat_update_32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
2485                                     pf->offset_loaded,
2486                                     &os->priority_xoff_tx[i],
2487                                     &ns->priority_xoff_tx[i]);
2488                 i40e_stat_update_32(hw, I40E_GLPRT_RXON2OFFCNT(hw->port, i),
2489                                     pf->offset_loaded,
2490                                     &os->priority_xon_2_xoff[i],
2491                                     &ns->priority_xon_2_xoff[i]);
2492         }
2493         i40e_stat_update_48(hw, I40E_GLPRT_PRC64H(hw->port),
2494                             I40E_GLPRT_PRC64L(hw->port),
2495                             pf->offset_loaded, &os->rx_size_64,
2496                             &ns->rx_size_64);
2497         i40e_stat_update_48(hw, I40E_GLPRT_PRC127H(hw->port),
2498                             I40E_GLPRT_PRC127L(hw->port),
2499                             pf->offset_loaded, &os->rx_size_127,
2500                             &ns->rx_size_127);
2501         i40e_stat_update_48(hw, I40E_GLPRT_PRC255H(hw->port),
2502                             I40E_GLPRT_PRC255L(hw->port),
2503                             pf->offset_loaded, &os->rx_size_255,
2504                             &ns->rx_size_255);
2505         i40e_stat_update_48(hw, I40E_GLPRT_PRC511H(hw->port),
2506                             I40E_GLPRT_PRC511L(hw->port),
2507                             pf->offset_loaded, &os->rx_size_511,
2508                             &ns->rx_size_511);
2509         i40e_stat_update_48(hw, I40E_GLPRT_PRC1023H(hw->port),
2510                             I40E_GLPRT_PRC1023L(hw->port),
2511                             pf->offset_loaded, &os->rx_size_1023,
2512                             &ns->rx_size_1023);
2513         i40e_stat_update_48(hw, I40E_GLPRT_PRC1522H(hw->port),
2514                             I40E_GLPRT_PRC1522L(hw->port),
2515                             pf->offset_loaded, &os->rx_size_1522,
2516                             &ns->rx_size_1522);
2517         i40e_stat_update_48(hw, I40E_GLPRT_PRC9522H(hw->port),
2518                             I40E_GLPRT_PRC9522L(hw->port),
2519                             pf->offset_loaded, &os->rx_size_big,
2520                             &ns->rx_size_big);
2521         i40e_stat_update_32(hw, I40E_GLPRT_RUC(hw->port),
2522                             pf->offset_loaded, &os->rx_undersize,
2523                             &ns->rx_undersize);
2524         i40e_stat_update_32(hw, I40E_GLPRT_RFC(hw->port),
2525                             pf->offset_loaded, &os->rx_fragments,
2526                             &ns->rx_fragments);
2527         i40e_stat_update_32(hw, I40E_GLPRT_ROC(hw->port),
2528                             pf->offset_loaded, &os->rx_oversize,
2529                             &ns->rx_oversize);
2530         i40e_stat_update_32(hw, I40E_GLPRT_RJC(hw->port),
2531                             pf->offset_loaded, &os->rx_jabber,
2532                             &ns->rx_jabber);
2533         i40e_stat_update_48(hw, I40E_GLPRT_PTC64H(hw->port),
2534                             I40E_GLPRT_PTC64L(hw->port),
2535                             pf->offset_loaded, &os->tx_size_64,
2536                             &ns->tx_size_64);
2537         i40e_stat_update_48(hw, I40E_GLPRT_PTC127H(hw->port),
2538                             I40E_GLPRT_PTC127L(hw->port),
2539                             pf->offset_loaded, &os->tx_size_127,
2540                             &ns->tx_size_127);
2541         i40e_stat_update_48(hw, I40E_GLPRT_PTC255H(hw->port),
2542                             I40E_GLPRT_PTC255L(hw->port),
2543                             pf->offset_loaded, &os->tx_size_255,
2544                             &ns->tx_size_255);
2545         i40e_stat_update_48(hw, I40E_GLPRT_PTC511H(hw->port),
2546                             I40E_GLPRT_PTC511L(hw->port),
2547                             pf->offset_loaded, &os->tx_size_511,
2548                             &ns->tx_size_511);
2549         i40e_stat_update_48(hw, I40E_GLPRT_PTC1023H(hw->port),
2550                             I40E_GLPRT_PTC1023L(hw->port),
2551                             pf->offset_loaded, &os->tx_size_1023,
2552                             &ns->tx_size_1023);
2553         i40e_stat_update_48(hw, I40E_GLPRT_PTC1522H(hw->port),
2554                             I40E_GLPRT_PTC1522L(hw->port),
2555                             pf->offset_loaded, &os->tx_size_1522,
2556                             &ns->tx_size_1522);
2557         i40e_stat_update_48(hw, I40E_GLPRT_PTC9522H(hw->port),
2558                             I40E_GLPRT_PTC9522L(hw->port),
2559                             pf->offset_loaded, &os->tx_size_big,
2560                             &ns->tx_size_big);
2561         i40e_stat_update_32(hw, I40E_GLQF_PCNT(pf->fdir.match_counter_index),
2562                            pf->offset_loaded,
2563                            &os->fd_sb_match, &ns->fd_sb_match);
2564         /* GLPRT_MSPDC not supported */
2565         /* GLPRT_XEC not supported */
2566
2567         pf->offset_loaded = true;
2568
2569         if (pf->main_vsi)
2570                 i40e_update_vsi_stats(pf->main_vsi);
2571 }
2572
2573 /* Get all statistics of a port */
2574 static void
2575 i40e_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
2576 {
2577         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2578         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2579         struct i40e_hw_port_stats *ns = &pf->stats; /* new stats */
2580         unsigned i;
2581
2582         /* call read registers - updates values, now write them to struct */
2583         i40e_read_stats_registers(pf, hw);
2584
2585         stats->ipackets = pf->main_vsi->eth_stats.rx_unicast +
2586                         pf->main_vsi->eth_stats.rx_multicast +
2587                         pf->main_vsi->eth_stats.rx_broadcast -
2588                         pf->main_vsi->eth_stats.rx_discards;
2589         stats->opackets = pf->main_vsi->eth_stats.tx_unicast +
2590                         pf->main_vsi->eth_stats.tx_multicast +
2591                         pf->main_vsi->eth_stats.tx_broadcast;
2592         stats->ibytes   = ns->eth.rx_bytes;
2593         stats->obytes   = ns->eth.tx_bytes;
2594         stats->oerrors  = ns->eth.tx_errors +
2595                         pf->main_vsi->eth_stats.tx_errors;
2596
2597         /* Rx Errors */
2598         stats->imissed  = ns->eth.rx_discards +
2599                         pf->main_vsi->eth_stats.rx_discards;
2600         stats->ierrors  = ns->crc_errors +
2601                         ns->rx_length_errors + ns->rx_undersize +
2602                         ns->rx_oversize + ns->rx_fragments + ns->rx_jabber;
2603
2604         PMD_DRV_LOG(DEBUG, "***************** PF stats start *******************");
2605         PMD_DRV_LOG(DEBUG, "rx_bytes:            %"PRIu64"", ns->eth.rx_bytes);
2606         PMD_DRV_LOG(DEBUG, "rx_unicast:          %"PRIu64"", ns->eth.rx_unicast);
2607         PMD_DRV_LOG(DEBUG, "rx_multicast:        %"PRIu64"", ns->eth.rx_multicast);
2608         PMD_DRV_LOG(DEBUG, "rx_broadcast:        %"PRIu64"", ns->eth.rx_broadcast);
2609         PMD_DRV_LOG(DEBUG, "rx_discards:         %"PRIu64"", ns->eth.rx_discards);
2610         PMD_DRV_LOG(DEBUG, "rx_unknown_protocol: %"PRIu64"",
2611                     ns->eth.rx_unknown_protocol);
2612         PMD_DRV_LOG(DEBUG, "tx_bytes:            %"PRIu64"", ns->eth.tx_bytes);
2613         PMD_DRV_LOG(DEBUG, "tx_unicast:          %"PRIu64"", ns->eth.tx_unicast);
2614         PMD_DRV_LOG(DEBUG, "tx_multicast:        %"PRIu64"", ns->eth.tx_multicast);
2615         PMD_DRV_LOG(DEBUG, "tx_broadcast:        %"PRIu64"", ns->eth.tx_broadcast);
2616         PMD_DRV_LOG(DEBUG, "tx_discards:         %"PRIu64"", ns->eth.tx_discards);
2617         PMD_DRV_LOG(DEBUG, "tx_errors:           %"PRIu64"", ns->eth.tx_errors);
2618
2619         PMD_DRV_LOG(DEBUG, "tx_dropped_link_down:     %"PRIu64"",
2620                     ns->tx_dropped_link_down);
2621         PMD_DRV_LOG(DEBUG, "crc_errors:               %"PRIu64"", ns->crc_errors);
2622         PMD_DRV_LOG(DEBUG, "illegal_bytes:            %"PRIu64"",
2623                     ns->illegal_bytes);
2624         PMD_DRV_LOG(DEBUG, "error_bytes:              %"PRIu64"", ns->error_bytes);
2625         PMD_DRV_LOG(DEBUG, "mac_local_faults:         %"PRIu64"",
2626                     ns->mac_local_faults);
2627         PMD_DRV_LOG(DEBUG, "mac_remote_faults:        %"PRIu64"",
2628                     ns->mac_remote_faults);
2629         PMD_DRV_LOG(DEBUG, "rx_length_errors:         %"PRIu64"",
2630                     ns->rx_length_errors);
2631         PMD_DRV_LOG(DEBUG, "link_xon_rx:              %"PRIu64"", ns->link_xon_rx);
2632         PMD_DRV_LOG(DEBUG, "link_xoff_rx:             %"PRIu64"", ns->link_xoff_rx);
2633         for (i = 0; i < 8; i++) {
2634                 PMD_DRV_LOG(DEBUG, "priority_xon_rx[%d]:      %"PRIu64"",
2635                                 i, ns->priority_xon_rx[i]);
2636                 PMD_DRV_LOG(DEBUG, "priority_xoff_rx[%d]:     %"PRIu64"",
2637                                 i, ns->priority_xoff_rx[i]);
2638         }
2639         PMD_DRV_LOG(DEBUG, "link_xon_tx:              %"PRIu64"", ns->link_xon_tx);
2640         PMD_DRV_LOG(DEBUG, "link_xoff_tx:             %"PRIu64"", ns->link_xoff_tx);
2641         for (i = 0; i < 8; i++) {
2642                 PMD_DRV_LOG(DEBUG, "priority_xon_tx[%d]:      %"PRIu64"",
2643                                 i, ns->priority_xon_tx[i]);
2644                 PMD_DRV_LOG(DEBUG, "priority_xoff_tx[%d]:     %"PRIu64"",
2645                                 i, ns->priority_xoff_tx[i]);
2646                 PMD_DRV_LOG(DEBUG, "priority_xon_2_xoff[%d]:  %"PRIu64"",
2647                                 i, ns->priority_xon_2_xoff[i]);
2648         }
2649         PMD_DRV_LOG(DEBUG, "rx_size_64:               %"PRIu64"", ns->rx_size_64);
2650         PMD_DRV_LOG(DEBUG, "rx_size_127:              %"PRIu64"", ns->rx_size_127);
2651         PMD_DRV_LOG(DEBUG, "rx_size_255:              %"PRIu64"", ns->rx_size_255);
2652         PMD_DRV_LOG(DEBUG, "rx_size_511:              %"PRIu64"", ns->rx_size_511);
2653         PMD_DRV_LOG(DEBUG, "rx_size_1023:             %"PRIu64"", ns->rx_size_1023);
2654         PMD_DRV_LOG(DEBUG, "rx_size_1522:             %"PRIu64"", ns->rx_size_1522);
2655         PMD_DRV_LOG(DEBUG, "rx_size_big:              %"PRIu64"", ns->rx_size_big);
2656         PMD_DRV_LOG(DEBUG, "rx_undersize:             %"PRIu64"", ns->rx_undersize);
2657         PMD_DRV_LOG(DEBUG, "rx_fragments:             %"PRIu64"", ns->rx_fragments);
2658         PMD_DRV_LOG(DEBUG, "rx_oversize:              %"PRIu64"", ns->rx_oversize);
2659         PMD_DRV_LOG(DEBUG, "rx_jabber:                %"PRIu64"", ns->rx_jabber);
2660         PMD_DRV_LOG(DEBUG, "tx_size_64:               %"PRIu64"", ns->tx_size_64);
2661         PMD_DRV_LOG(DEBUG, "tx_size_127:              %"PRIu64"", ns->tx_size_127);
2662         PMD_DRV_LOG(DEBUG, "tx_size_255:              %"PRIu64"", ns->tx_size_255);
2663         PMD_DRV_LOG(DEBUG, "tx_size_511:              %"PRIu64"", ns->tx_size_511);
2664         PMD_DRV_LOG(DEBUG, "tx_size_1023:             %"PRIu64"", ns->tx_size_1023);
2665         PMD_DRV_LOG(DEBUG, "tx_size_1522:             %"PRIu64"", ns->tx_size_1522);
2666         PMD_DRV_LOG(DEBUG, "tx_size_big:              %"PRIu64"", ns->tx_size_big);
2667         PMD_DRV_LOG(DEBUG, "mac_short_packet_dropped: %"PRIu64"",
2668                         ns->mac_short_packet_dropped);
2669         PMD_DRV_LOG(DEBUG, "checksum_error:           %"PRIu64"",
2670                     ns->checksum_error);
2671         PMD_DRV_LOG(DEBUG, "fdir_match:               %"PRIu64"", ns->fd_sb_match);
2672         PMD_DRV_LOG(DEBUG, "***************** PF stats end ********************");
2673 }
2674
2675 /* Reset the statistics */
2676 static void
2677 i40e_dev_stats_reset(struct rte_eth_dev *dev)
2678 {
2679         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2680         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2681
2682         /* Mark PF and VSI stats to update the offset, aka "reset" */
2683         pf->offset_loaded = false;
2684         if (pf->main_vsi)
2685                 pf->main_vsi->offset_loaded = false;
2686
2687         /* read the stats, reading current register values into offset */
2688         i40e_read_stats_registers(pf, hw);
2689 }
2690
2691 static uint32_t
2692 i40e_xstats_calc_num(void)
2693 {
2694         return I40E_NB_ETH_XSTATS + I40E_NB_HW_PORT_XSTATS +
2695                 (I40E_NB_RXQ_PRIO_XSTATS * 8) +
2696                 (I40E_NB_TXQ_PRIO_XSTATS * 8);
2697 }
2698
2699 static int i40e_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
2700                                      struct rte_eth_xstat_name *xstats_names,
2701                                      __rte_unused unsigned limit)
2702 {
2703         unsigned count = 0;
2704         unsigned i, prio;
2705
2706         if (xstats_names == NULL)
2707                 return i40e_xstats_calc_num();
2708
2709         /* Note: limit checked in rte_eth_xstats_names() */
2710
2711         /* Get stats from i40e_eth_stats struct */
2712         for (i = 0; i < I40E_NB_ETH_XSTATS; i++) {
2713                 snprintf(xstats_names[count].name,
2714                          sizeof(xstats_names[count].name),
2715                          "%s", rte_i40e_stats_strings[i].name);
2716                 count++;
2717         }
2718
2719         /* Get individiual stats from i40e_hw_port struct */
2720         for (i = 0; i < I40E_NB_HW_PORT_XSTATS; i++) {
2721                 snprintf(xstats_names[count].name,
2722                         sizeof(xstats_names[count].name),
2723                          "%s", rte_i40e_hw_port_strings[i].name);
2724                 count++;
2725         }
2726
2727         for (i = 0; i < I40E_NB_RXQ_PRIO_XSTATS; i++) {
2728                 for (prio = 0; prio < 8; prio++) {
2729                         snprintf(xstats_names[count].name,
2730                                  sizeof(xstats_names[count].name),
2731                                  "rx_priority%u_%s", prio,
2732                                  rte_i40e_rxq_prio_strings[i].name);
2733                         count++;
2734                 }
2735         }
2736
2737         for (i = 0; i < I40E_NB_TXQ_PRIO_XSTATS; i++) {
2738                 for (prio = 0; prio < 8; prio++) {
2739                         snprintf(xstats_names[count].name,
2740                                  sizeof(xstats_names[count].name),
2741                                  "tx_priority%u_%s", prio,
2742                                  rte_i40e_txq_prio_strings[i].name);
2743                         count++;
2744                 }
2745         }
2746         return count;
2747 }
2748
2749 static int
2750 i40e_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
2751                     unsigned n)
2752 {
2753         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2754         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2755         unsigned i, count, prio;
2756         struct i40e_hw_port_stats *hw_stats = &pf->stats;
2757
2758         count = i40e_xstats_calc_num();
2759         if (n < count)
2760                 return count;
2761
2762         i40e_read_stats_registers(pf, hw);
2763
2764         if (xstats == NULL)
2765                 return 0;
2766
2767         count = 0;
2768
2769         /* Get stats from i40e_eth_stats struct */
2770         for (i = 0; i < I40E_NB_ETH_XSTATS; i++) {
2771                 xstats[count].value = *(uint64_t *)(((char *)&hw_stats->eth) +
2772                         rte_i40e_stats_strings[i].offset);
2773                 xstats[count].id = count;
2774                 count++;
2775         }
2776
2777         /* Get individiual stats from i40e_hw_port struct */
2778         for (i = 0; i < I40E_NB_HW_PORT_XSTATS; i++) {
2779                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
2780                         rte_i40e_hw_port_strings[i].offset);
2781                 xstats[count].id = count;
2782                 count++;
2783         }
2784
2785         for (i = 0; i < I40E_NB_RXQ_PRIO_XSTATS; i++) {
2786                 for (prio = 0; prio < 8; prio++) {
2787                         xstats[count].value =
2788                                 *(uint64_t *)(((char *)hw_stats) +
2789                                 rte_i40e_rxq_prio_strings[i].offset +
2790                                 (sizeof(uint64_t) * prio));
2791                         xstats[count].id = count;
2792                         count++;
2793                 }
2794         }
2795
2796         for (i = 0; i < I40E_NB_TXQ_PRIO_XSTATS; i++) {
2797                 for (prio = 0; prio < 8; prio++) {
2798                         xstats[count].value =
2799                                 *(uint64_t *)(((char *)hw_stats) +
2800                                 rte_i40e_txq_prio_strings[i].offset +
2801                                 (sizeof(uint64_t) * prio));
2802                         xstats[count].id = count;
2803                         count++;
2804                 }
2805         }
2806
2807         return count;
2808 }
2809
2810 static int
2811 i40e_dev_queue_stats_mapping_set(__rte_unused struct rte_eth_dev *dev,
2812                                  __rte_unused uint16_t queue_id,
2813                                  __rte_unused uint8_t stat_idx,
2814                                  __rte_unused uint8_t is_rx)
2815 {
2816         PMD_INIT_FUNC_TRACE();
2817
2818         return -ENOSYS;
2819 }
2820
2821 static void
2822 i40e_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
2823 {
2824         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2825         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2826         struct i40e_vsi *vsi = pf->main_vsi;
2827
2828         dev_info->max_rx_queues = vsi->nb_qps;
2829         dev_info->max_tx_queues = vsi->nb_qps;
2830         dev_info->min_rx_bufsize = I40E_BUF_SIZE_MIN;
2831         dev_info->max_rx_pktlen = I40E_FRAME_SIZE_MAX;
2832         dev_info->max_mac_addrs = vsi->max_macaddrs;
2833         dev_info->max_vfs = dev->pci_dev->max_vfs;
2834         dev_info->rx_offload_capa =
2835                 DEV_RX_OFFLOAD_VLAN_STRIP |
2836                 DEV_RX_OFFLOAD_QINQ_STRIP |
2837                 DEV_RX_OFFLOAD_IPV4_CKSUM |
2838                 DEV_RX_OFFLOAD_UDP_CKSUM |
2839                 DEV_RX_OFFLOAD_TCP_CKSUM;
2840         dev_info->tx_offload_capa =
2841                 DEV_TX_OFFLOAD_VLAN_INSERT |
2842                 DEV_TX_OFFLOAD_QINQ_INSERT |
2843                 DEV_TX_OFFLOAD_IPV4_CKSUM |
2844                 DEV_TX_OFFLOAD_UDP_CKSUM |
2845                 DEV_TX_OFFLOAD_TCP_CKSUM |
2846                 DEV_TX_OFFLOAD_SCTP_CKSUM |
2847                 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
2848                 DEV_TX_OFFLOAD_TCP_TSO |
2849                 DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
2850                 DEV_TX_OFFLOAD_GRE_TNL_TSO |
2851                 DEV_TX_OFFLOAD_IPIP_TNL_TSO |
2852                 DEV_TX_OFFLOAD_GENEVE_TNL_TSO;
2853         dev_info->hash_key_size = (I40E_PFQF_HKEY_MAX_INDEX + 1) *
2854                                                 sizeof(uint32_t);
2855         dev_info->reta_size = pf->hash_lut_size;
2856         dev_info->flow_type_rss_offloads = I40E_RSS_OFFLOAD_ALL;
2857
2858         dev_info->default_rxconf = (struct rte_eth_rxconf) {
2859                 .rx_thresh = {
2860                         .pthresh = I40E_DEFAULT_RX_PTHRESH,
2861                         .hthresh = I40E_DEFAULT_RX_HTHRESH,
2862                         .wthresh = I40E_DEFAULT_RX_WTHRESH,
2863                 },
2864                 .rx_free_thresh = I40E_DEFAULT_RX_FREE_THRESH,
2865                 .rx_drop_en = 0,
2866         };
2867
2868         dev_info->default_txconf = (struct rte_eth_txconf) {
2869                 .tx_thresh = {
2870                         .pthresh = I40E_DEFAULT_TX_PTHRESH,
2871                         .hthresh = I40E_DEFAULT_TX_HTHRESH,
2872                         .wthresh = I40E_DEFAULT_TX_WTHRESH,
2873                 },
2874                 .tx_free_thresh = I40E_DEFAULT_TX_FREE_THRESH,
2875                 .tx_rs_thresh = I40E_DEFAULT_TX_RSBIT_THRESH,
2876                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
2877                                 ETH_TXQ_FLAGS_NOOFFLOADS,
2878         };
2879
2880         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
2881                 .nb_max = I40E_MAX_RING_DESC,
2882                 .nb_min = I40E_MIN_RING_DESC,
2883                 .nb_align = I40E_ALIGN_RING_DESC,
2884         };
2885
2886         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
2887                 .nb_max = I40E_MAX_RING_DESC,
2888                 .nb_min = I40E_MIN_RING_DESC,
2889                 .nb_align = I40E_ALIGN_RING_DESC,
2890         };
2891
2892         if (pf->flags & I40E_FLAG_VMDQ) {
2893                 dev_info->max_vmdq_pools = pf->max_nb_vmdq_vsi;
2894                 dev_info->vmdq_queue_base = dev_info->max_rx_queues;
2895                 dev_info->vmdq_queue_num = pf->vmdq_nb_qps *
2896                                                 pf->max_nb_vmdq_vsi;
2897                 dev_info->vmdq_pool_base = I40E_VMDQ_POOL_BASE;
2898                 dev_info->max_rx_queues += dev_info->vmdq_queue_num;
2899                 dev_info->max_tx_queues += dev_info->vmdq_queue_num;
2900         }
2901
2902         if (I40E_PHY_TYPE_SUPPORT_40G(hw->phy.phy_types))
2903                 /* For XL710 */
2904                 dev_info->speed_capa = ETH_LINK_SPEED_40G;
2905         else if (I40E_PHY_TYPE_SUPPORT_25G(hw->phy.phy_types))
2906                 /* For XXV710 */
2907                 dev_info->speed_capa = ETH_LINK_SPEED_25G;
2908         else
2909                 /* For X710 */
2910                 dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G;
2911 }
2912
2913 static int
2914 i40e_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2915 {
2916         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2917         struct i40e_vsi *vsi = pf->main_vsi;
2918         PMD_INIT_FUNC_TRACE();
2919
2920         if (on)
2921                 return i40e_vsi_add_vlan(vsi, vlan_id);
2922         else
2923                 return i40e_vsi_delete_vlan(vsi, vlan_id);
2924 }
2925
2926 static int
2927 i40e_vlan_tpid_set(struct rte_eth_dev *dev,
2928                    enum rte_vlan_type vlan_type,
2929                    uint16_t tpid)
2930 {
2931         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2932         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2933         uint64_t reg_r = 0, reg_w = 0;
2934         uint16_t reg_id = 0;
2935         int ret = 0;
2936         int qinq = dev->data->dev_conf.rxmode.hw_vlan_extend;
2937
2938         if (pf->support_multi_driver) {
2939                 PMD_DRV_LOG(ERR, "Setting TPID is not supported.");
2940                 return -ENOTSUP;
2941         }
2942
2943         switch (vlan_type) {
2944         case ETH_VLAN_TYPE_OUTER:
2945                 if (qinq)
2946                         reg_id = 2;
2947                 else
2948                         reg_id = 3;
2949                 break;
2950         case ETH_VLAN_TYPE_INNER:
2951                 if (qinq)
2952                         reg_id = 3;
2953                 else {
2954                         ret = -EINVAL;
2955                         PMD_DRV_LOG(ERR,
2956                                 "Unsupported vlan type in single vlan.\n");
2957                         return ret;
2958                 }
2959                 break;
2960         default:
2961                 ret = -EINVAL;
2962                 PMD_DRV_LOG(ERR, "Unsupported vlan type %d", vlan_type);
2963                 return ret;
2964         }
2965         ret = i40e_aq_debug_read_register(hw, I40E_GL_SWT_L2TAGCTRL(reg_id),
2966                                           &reg_r, NULL);
2967         if (ret != I40E_SUCCESS) {
2968                 PMD_DRV_LOG(ERR, "Fail to debug read from "
2969                             "I40E_GL_SWT_L2TAGCTRL[%d]", reg_id);
2970                 ret = -EIO;
2971                 return ret;
2972         }
2973         PMD_DRV_LOG(DEBUG, "Debug read from I40E_GL_SWT_L2TAGCTRL[%d]: "
2974                     "0x%08"PRIx64"", reg_id, reg_r);
2975
2976         reg_w = reg_r & (~(I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_MASK));
2977         reg_w |= ((uint64_t)tpid << I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_SHIFT);
2978         if (reg_r == reg_w) {
2979                 ret = 0;
2980                 PMD_DRV_LOG(DEBUG, "No need to write");
2981                 return ret;
2982         }
2983
2984         ret = i40e_aq_debug_write_register(hw, I40E_GL_SWT_L2TAGCTRL(reg_id),
2985                                            reg_w, NULL);
2986         if (ret != I40E_SUCCESS) {
2987                 ret = -EIO;
2988                 PMD_DRV_LOG(ERR, "Fail to debug write to "
2989                             "I40E_GL_SWT_L2TAGCTRL[%d]", reg_id);
2990                 return ret;
2991         }
2992         PMD_DRV_LOG(DEBUG,
2993                     "Global register 0x%08x is changed with value 0x%08x",
2994                     I40E_GL_SWT_L2TAGCTRL(reg_id), (uint32_t)reg_w);
2995
2996         i40e_global_cfg_warning(I40E_WARNING_TPID);
2997
2998         return ret;
2999 }
3000
3001 static void
3002 i40e_vlan_offload_set(struct rte_eth_dev *dev, int mask)
3003 {
3004         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3005         struct i40e_vsi *vsi = pf->main_vsi;
3006
3007         if (mask & ETH_VLAN_FILTER_MASK) {
3008                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
3009                         i40e_vsi_config_vlan_filter(vsi, TRUE);
3010                 else
3011                         i40e_vsi_config_vlan_filter(vsi, FALSE);
3012         }
3013
3014         if (mask & ETH_VLAN_STRIP_MASK) {
3015                 /* Enable or disable VLAN stripping */
3016                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
3017                         i40e_vsi_config_vlan_stripping(vsi, TRUE);
3018                 else
3019                         i40e_vsi_config_vlan_stripping(vsi, FALSE);
3020         }
3021
3022         if (mask & ETH_VLAN_EXTEND_MASK) {
3023                 if (dev->data->dev_conf.rxmode.hw_vlan_extend) {
3024                         i40e_vsi_config_double_vlan(vsi, TRUE);
3025                         /* Set global registers with default ether type value */
3026                         i40e_vlan_tpid_set(dev, ETH_VLAN_TYPE_OUTER,
3027                                            ETHER_TYPE_VLAN);
3028                         i40e_vlan_tpid_set(dev, ETH_VLAN_TYPE_INNER,
3029                                            ETHER_TYPE_VLAN);
3030                 }
3031                 else
3032                         i40e_vsi_config_double_vlan(vsi, FALSE);
3033         }
3034 }
3035
3036 static void
3037 i40e_vlan_strip_queue_set(__rte_unused struct rte_eth_dev *dev,
3038                           __rte_unused uint16_t queue,
3039                           __rte_unused int on)
3040 {
3041         PMD_INIT_FUNC_TRACE();
3042 }
3043
3044 static int
3045 i40e_vlan_pvid_set(struct rte_eth_dev *dev, uint16_t pvid, int on)
3046 {
3047         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3048         struct i40e_vsi *vsi = pf->main_vsi;
3049         struct rte_eth_dev_data *data = I40E_VSI_TO_DEV_DATA(vsi);
3050         struct i40e_vsi_vlan_pvid_info info;
3051
3052         memset(&info, 0, sizeof(info));
3053         info.on = on;
3054         if (info.on)
3055                 info.config.pvid = pvid;
3056         else {
3057                 info.config.reject.tagged =
3058                                 data->dev_conf.txmode.hw_vlan_reject_tagged;
3059                 info.config.reject.untagged =
3060                                 data->dev_conf.txmode.hw_vlan_reject_untagged;
3061         }
3062
3063         return i40e_vsi_vlan_pvid_set(vsi, &info);
3064 }
3065
3066 static int
3067 i40e_dev_led_on(struct rte_eth_dev *dev)
3068 {
3069         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3070         uint32_t mode = i40e_led_get(hw);
3071
3072         if (mode == 0)
3073                 i40e_led_set(hw, 0xf, true); /* 0xf means led always true */
3074
3075         return 0;
3076 }
3077
3078 static int
3079 i40e_dev_led_off(struct rte_eth_dev *dev)
3080 {
3081         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3082         uint32_t mode = i40e_led_get(hw);
3083
3084         if (mode != 0)
3085                 i40e_led_set(hw, 0, false);
3086
3087         return 0;
3088 }
3089
3090 static int
3091 i40e_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
3092 {
3093         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3094         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3095
3096         fc_conf->pause_time = pf->fc_conf.pause_time;
3097
3098         /* read out from register, in case they are modified by other port */
3099         pf->fc_conf.high_water[I40E_MAX_TRAFFIC_CLASS] =
3100                 I40E_READ_REG(hw, I40E_GLRPB_GHW) >> I40E_KILOSHIFT;
3101         pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS] =
3102                 I40E_READ_REG(hw, I40E_GLRPB_GLW) >> I40E_KILOSHIFT;
3103
3104         fc_conf->high_water =  pf->fc_conf.high_water[I40E_MAX_TRAFFIC_CLASS];
3105         fc_conf->low_water = pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS];
3106
3107          /* Return current mode according to actual setting*/
3108         switch (hw->fc.current_mode) {
3109         case I40E_FC_FULL:
3110                 fc_conf->mode = RTE_FC_FULL;
3111                 break;
3112         case I40E_FC_TX_PAUSE:
3113                 fc_conf->mode = RTE_FC_TX_PAUSE;
3114                 break;
3115         case I40E_FC_RX_PAUSE:
3116                 fc_conf->mode = RTE_FC_RX_PAUSE;
3117                 break;
3118         case I40E_FC_NONE:
3119         default:
3120                 fc_conf->mode = RTE_FC_NONE;
3121         };
3122
3123         return 0;
3124 }
3125
3126 static int
3127 i40e_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
3128 {
3129         uint32_t mflcn_reg, fctrl_reg, reg;
3130         uint32_t max_high_water;
3131         uint8_t i, aq_failure;
3132         int err;
3133         struct i40e_hw *hw;
3134         struct i40e_pf *pf;
3135         enum i40e_fc_mode rte_fcmode_2_i40e_fcmode[] = {
3136                 [RTE_FC_NONE] = I40E_FC_NONE,
3137                 [RTE_FC_RX_PAUSE] = I40E_FC_RX_PAUSE,
3138                 [RTE_FC_TX_PAUSE] = I40E_FC_TX_PAUSE,
3139                 [RTE_FC_FULL] = I40E_FC_FULL
3140         };
3141
3142         /* high_water field in the rte_eth_fc_conf using the kilobytes unit */
3143
3144         max_high_water = I40E_RXPBSIZE >> I40E_KILOSHIFT;
3145         if ((fc_conf->high_water > max_high_water) ||
3146                         (fc_conf->high_water < fc_conf->low_water)) {
3147                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB, "
3148                         "High_water must <= %d.", max_high_water);
3149                 return -EINVAL;
3150         }
3151
3152         hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3153         pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3154         hw->fc.requested_mode = rte_fcmode_2_i40e_fcmode[fc_conf->mode];
3155
3156         pf->fc_conf.pause_time = fc_conf->pause_time;
3157         pf->fc_conf.high_water[I40E_MAX_TRAFFIC_CLASS] = fc_conf->high_water;
3158         pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS] = fc_conf->low_water;
3159
3160         PMD_INIT_FUNC_TRACE();
3161
3162         /* All the link flow control related enable/disable register
3163          * configuration is handle by the F/W
3164          */
3165         err = i40e_set_fc(hw, &aq_failure, true);
3166         if (err < 0)
3167                 return -ENOSYS;
3168
3169         if (I40E_PHY_TYPE_SUPPORT_40G(hw->phy.phy_types)) {
3170                 /* Configure flow control refresh threshold,
3171                  * the value for stat_tx_pause_refresh_timer[8]
3172                  * is used for global pause operation.
3173                  */
3174
3175                 I40E_WRITE_REG(hw,
3176                                I40E_PRTMAC_HSEC_CTL_TX_PAUSE_REFRESH_TIMER(8),
3177                                pf->fc_conf.pause_time);
3178
3179                 /* configure the timer value included in transmitted pause
3180                  * frame,
3181                  * the value for stat_tx_pause_quanta[8] is used for global
3182                  * pause operation
3183                  */
3184                 I40E_WRITE_REG(hw, I40E_PRTMAC_HSEC_CTL_TX_PAUSE_QUANTA(8),
3185                                pf->fc_conf.pause_time);
3186
3187                 fctrl_reg = I40E_READ_REG(hw,
3188                                           I40E_PRTMAC_HSEC_CTL_RX_FORWARD_CONTROL);
3189
3190                 if (fc_conf->mac_ctrl_frame_fwd != 0)
3191                         fctrl_reg |= I40E_PRTMAC_FWD_CTRL;
3192                 else
3193                         fctrl_reg &= ~I40E_PRTMAC_FWD_CTRL;
3194
3195                 I40E_WRITE_REG(hw, I40E_PRTMAC_HSEC_CTL_RX_FORWARD_CONTROL,
3196                                fctrl_reg);
3197         } else {
3198                 /* Configure pause time (2 TCs per register) */
3199                 reg = (uint32_t)pf->fc_conf.pause_time * (uint32_t)0x00010001;
3200                 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS / 2; i++)
3201                         I40E_WRITE_REG(hw, I40E_PRTDCB_FCTTVN(i), reg);
3202
3203                 /* Configure flow control refresh threshold value */
3204                 I40E_WRITE_REG(hw, I40E_PRTDCB_FCRTV,
3205                                pf->fc_conf.pause_time / 2);
3206
3207                 mflcn_reg = I40E_READ_REG(hw, I40E_PRTDCB_MFLCN);
3208
3209                 /* set or clear MFLCN.PMCF & MFLCN.DPF bits
3210                  *depending on configuration
3211                  */
3212                 if (fc_conf->mac_ctrl_frame_fwd != 0) {
3213                         mflcn_reg |= I40E_PRTDCB_MFLCN_PMCF_MASK;
3214                         mflcn_reg &= ~I40E_PRTDCB_MFLCN_DPF_MASK;
3215                 } else {
3216                         mflcn_reg &= ~I40E_PRTDCB_MFLCN_PMCF_MASK;
3217                         mflcn_reg |= I40E_PRTDCB_MFLCN_DPF_MASK;
3218                 }
3219
3220                 I40E_WRITE_REG(hw, I40E_PRTDCB_MFLCN, mflcn_reg);
3221         }
3222
3223         if (!pf->support_multi_driver) {
3224                 /* config water marker both based on the packets and bytes */
3225                 I40E_WRITE_GLB_REG(hw, I40E_GLRPB_PHW,
3226                                 (pf->fc_conf.high_water[I40E_MAX_TRAFFIC_CLASS]
3227                                  << I40E_KILOSHIFT) / I40E_PACKET_AVERAGE_SIZE);
3228                 I40E_WRITE_GLB_REG(hw, I40E_GLRPB_PLW,
3229                                 (pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS]
3230                                  << I40E_KILOSHIFT) / I40E_PACKET_AVERAGE_SIZE);
3231                 I40E_WRITE_GLB_REG(hw, I40E_GLRPB_GHW,
3232                                  pf->fc_conf.high_water[I40E_MAX_TRAFFIC_CLASS]
3233                                  << I40E_KILOSHIFT);
3234                 I40E_WRITE_GLB_REG(hw, I40E_GLRPB_GLW,
3235                                   pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS]
3236                                   << I40E_KILOSHIFT);
3237                 i40e_global_cfg_warning(I40E_WARNING_FLOW_CTL);
3238         } else {
3239                 PMD_DRV_LOG(ERR,
3240                             "Water marker configuration is not supported.");
3241         }
3242
3243         I40E_WRITE_FLUSH(hw);
3244
3245         return 0;
3246 }
3247
3248 static int
3249 i40e_priority_flow_ctrl_set(__rte_unused struct rte_eth_dev *dev,
3250                             __rte_unused struct rte_eth_pfc_conf *pfc_conf)
3251 {
3252         PMD_INIT_FUNC_TRACE();
3253
3254         return -ENOSYS;
3255 }
3256
3257 /* Add a MAC address, and update filters */
3258 static void
3259 i40e_macaddr_add(struct rte_eth_dev *dev,
3260                  struct ether_addr *mac_addr,
3261                  __rte_unused uint32_t index,
3262                  uint32_t pool)
3263 {
3264         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3265         struct i40e_mac_filter_info mac_filter;
3266         struct i40e_vsi *vsi;
3267         int ret;
3268
3269         /* If VMDQ not enabled or configured, return */
3270         if (pool != 0 && (!(pf->flags & I40E_FLAG_VMDQ) ||
3271                           !pf->nb_cfg_vmdq_vsi)) {
3272                 PMD_DRV_LOG(ERR, "VMDQ not %s, can't set mac to pool %u",
3273                         pf->flags & I40E_FLAG_VMDQ ? "configured" : "enabled",
3274                         pool);
3275                 return;
3276         }
3277
3278         if (pool > pf->nb_cfg_vmdq_vsi) {
3279                 PMD_DRV_LOG(ERR, "Pool number %u invalid. Max pool is %u",
3280                                 pool, pf->nb_cfg_vmdq_vsi);
3281                 return;
3282         }
3283
3284         (void)rte_memcpy(&mac_filter.mac_addr, mac_addr, ETHER_ADDR_LEN);
3285         if (dev->data->dev_conf.rxmode.hw_vlan_filter)
3286                 mac_filter.filter_type = RTE_MACVLAN_PERFECT_MATCH;
3287         else
3288                 mac_filter.filter_type = RTE_MAC_PERFECT_MATCH;
3289
3290         if (pool == 0)
3291                 vsi = pf->main_vsi;
3292         else
3293                 vsi = pf->vmdq[pool - 1].vsi;
3294
3295         ret = i40e_vsi_add_mac(vsi, &mac_filter);
3296         if (ret != I40E_SUCCESS) {
3297                 PMD_DRV_LOG(ERR, "Failed to add MACVLAN filter");
3298                 return;
3299         }
3300 }
3301
3302 /* Remove a MAC address, and update filters */
3303 static void
3304 i40e_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
3305 {
3306         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3307         struct i40e_vsi *vsi;
3308         struct rte_eth_dev_data *data = dev->data;
3309         struct ether_addr *macaddr;
3310         int ret;
3311         uint32_t i;
3312         uint64_t pool_sel;
3313
3314         macaddr = &(data->mac_addrs[index]);
3315
3316         pool_sel = dev->data->mac_pool_sel[index];
3317
3318         for (i = 0; i < sizeof(pool_sel) * CHAR_BIT; i++) {
3319                 if (pool_sel & (1ULL << i)) {
3320                         if (i == 0)
3321                                 vsi = pf->main_vsi;
3322                         else {
3323                                 /* No VMDQ pool enabled or configured */
3324                                 if (!(pf->flags & I40E_FLAG_VMDQ) ||
3325                                         (i > pf->nb_cfg_vmdq_vsi)) {
3326                                         PMD_DRV_LOG(ERR, "No VMDQ pool enabled"
3327                                                         "/configured");
3328                                         return;
3329                                 }
3330                                 vsi = pf->vmdq[i - 1].vsi;
3331                         }
3332                         ret = i40e_vsi_delete_mac(vsi, macaddr);
3333
3334                         if (ret) {
3335                                 PMD_DRV_LOG(ERR, "Failed to remove MACVLAN filter");
3336                                 return;
3337                         }
3338                 }
3339         }
3340 }
3341
3342 /* Set perfect match or hash match of MAC and VLAN for a VF */
3343 static int
3344 i40e_vf_mac_filter_set(struct i40e_pf *pf,
3345                  struct rte_eth_mac_filter *filter,
3346                  bool add)
3347 {
3348         struct i40e_hw *hw;
3349         struct i40e_mac_filter_info mac_filter;
3350         struct ether_addr old_mac;
3351         struct ether_addr *new_mac;
3352         struct i40e_pf_vf *vf = NULL;
3353         uint16_t vf_id;
3354         int ret;
3355
3356         if (pf == NULL) {
3357                 PMD_DRV_LOG(ERR, "Invalid PF argument.");
3358                 return -EINVAL;
3359         }
3360         hw = I40E_PF_TO_HW(pf);
3361
3362         if (filter == NULL) {
3363                 PMD_DRV_LOG(ERR, "Invalid mac filter argument.");
3364                 return -EINVAL;
3365         }
3366
3367         new_mac = &filter->mac_addr;
3368
3369         if (is_zero_ether_addr(new_mac)) {
3370                 PMD_DRV_LOG(ERR, "Invalid ethernet address.");
3371                 return -EINVAL;
3372         }
3373
3374         vf_id = filter->dst_id;
3375
3376         if (vf_id > pf->vf_num - 1 || !pf->vfs) {
3377                 PMD_DRV_LOG(ERR, "Invalid argument.");
3378                 return -EINVAL;
3379         }
3380         vf = &pf->vfs[vf_id];
3381
3382         if (add && is_same_ether_addr(new_mac, &(pf->dev_addr))) {
3383                 PMD_DRV_LOG(INFO, "Ignore adding permanent MAC address.");
3384                 return -EINVAL;
3385         }
3386
3387         if (add) {
3388                 (void)rte_memcpy(&old_mac, hw->mac.addr, ETHER_ADDR_LEN);
3389                 (void)rte_memcpy(hw->mac.addr, new_mac->addr_bytes,
3390                                 ETHER_ADDR_LEN);
3391                 (void)rte_memcpy(&mac_filter.mac_addr, &filter->mac_addr,
3392                                  ETHER_ADDR_LEN);
3393
3394                 mac_filter.filter_type = filter->filter_type;
3395                 ret = i40e_vsi_add_mac(vf->vsi, &mac_filter);
3396                 if (ret != I40E_SUCCESS) {
3397                         PMD_DRV_LOG(ERR, "Failed to add MAC filter.");
3398                         return -1;
3399                 }
3400                 ether_addr_copy(new_mac, &pf->dev_addr);
3401         } else {
3402                 (void)rte_memcpy(hw->mac.addr, hw->mac.perm_addr,
3403                                 ETHER_ADDR_LEN);
3404                 ret = i40e_vsi_delete_mac(vf->vsi, &filter->mac_addr);
3405                 if (ret != I40E_SUCCESS) {
3406                         PMD_DRV_LOG(ERR, "Failed to delete MAC filter.");
3407                         return -1;
3408                 }
3409
3410                 /* Clear device address as it has been removed */
3411                 if (is_same_ether_addr(&(pf->dev_addr), new_mac))
3412                         memset(&pf->dev_addr, 0, sizeof(struct ether_addr));
3413         }
3414
3415         return 0;
3416 }
3417
3418 /* MAC filter handle */
3419 static int
3420 i40e_mac_filter_handle(struct rte_eth_dev *dev, enum rte_filter_op filter_op,
3421                 void *arg)
3422 {
3423         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3424         struct rte_eth_mac_filter *filter;
3425         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
3426         int ret = I40E_NOT_SUPPORTED;
3427
3428         filter = (struct rte_eth_mac_filter *)(arg);
3429
3430         switch (filter_op) {
3431         case RTE_ETH_FILTER_NOP:
3432                 ret = I40E_SUCCESS;
3433                 break;
3434         case RTE_ETH_FILTER_ADD:
3435                 i40e_pf_disable_irq0(hw);
3436                 if (filter->is_vf)
3437                         ret = i40e_vf_mac_filter_set(pf, filter, 1);
3438                 i40e_pf_enable_irq0(hw);
3439                 break;
3440         case RTE_ETH_FILTER_DELETE:
3441                 i40e_pf_disable_irq0(hw);
3442                 if (filter->is_vf)
3443                         ret = i40e_vf_mac_filter_set(pf, filter, 0);
3444                 i40e_pf_enable_irq0(hw);
3445                 break;
3446         default:
3447                 PMD_DRV_LOG(ERR, "unknown operation %u", filter_op);
3448                 ret = I40E_ERR_PARAM;
3449                 break;
3450         }
3451
3452         return ret;
3453 }
3454
3455 static int
3456 i40e_get_rss_lut(struct i40e_vsi *vsi, uint8_t *lut, uint16_t lut_size)
3457 {
3458         struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
3459         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
3460         int ret;
3461
3462         if (!lut)
3463                 return -EINVAL;
3464
3465         if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
3466                 ret = i40e_aq_get_rss_lut(hw, vsi->vsi_id, TRUE,
3467                                           lut, lut_size);
3468                 if (ret) {
3469                         PMD_DRV_LOG(ERR, "Failed to get RSS lookup table");
3470                         return ret;
3471                 }
3472         } else {
3473                 uint32_t *lut_dw = (uint32_t *)lut;
3474                 uint16_t i, lut_size_dw = lut_size / 4;
3475
3476                 for (i = 0; i < lut_size_dw; i++)
3477                         lut_dw[i] = I40E_READ_REG(hw, I40E_PFQF_HLUT(i));
3478         }
3479
3480         return 0;
3481 }
3482
3483 static int
3484 i40e_set_rss_lut(struct i40e_vsi *vsi, uint8_t *lut, uint16_t lut_size)
3485 {
3486         struct i40e_pf *pf;
3487         struct i40e_hw *hw;
3488         int ret;
3489
3490         if (!vsi || !lut)
3491                 return -EINVAL;
3492
3493         pf = I40E_VSI_TO_PF(vsi);
3494         hw = I40E_VSI_TO_HW(vsi);
3495
3496         if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
3497                 ret = i40e_aq_set_rss_lut(hw, vsi->vsi_id, TRUE,
3498                                           lut, lut_size);
3499                 if (ret) {
3500                         PMD_DRV_LOG(ERR, "Failed to set RSS lookup table");
3501                         return ret;
3502                 }
3503         } else {
3504                 uint32_t *lut_dw = (uint32_t *)lut;
3505                 uint16_t i, lut_size_dw = lut_size / 4;
3506
3507                 for (i = 0; i < lut_size_dw; i++)
3508                         I40E_WRITE_REG(hw, I40E_PFQF_HLUT(i), lut_dw[i]);
3509                 I40E_WRITE_FLUSH(hw);
3510         }
3511
3512         return 0;
3513 }
3514
3515 static int
3516 i40e_dev_rss_reta_update(struct rte_eth_dev *dev,
3517                          struct rte_eth_rss_reta_entry64 *reta_conf,
3518                          uint16_t reta_size)
3519 {
3520         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3521         uint16_t i, lut_size = pf->hash_lut_size;
3522         uint16_t idx, shift;
3523         uint8_t *lut;
3524         int ret;
3525
3526         if (reta_size != lut_size ||
3527                 reta_size > ETH_RSS_RETA_SIZE_512) {
3528                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
3529                         "(%d) doesn't match the number hardware can supported "
3530                                         "(%d)\n", reta_size, lut_size);
3531                 return -EINVAL;
3532         }
3533
3534         lut = rte_zmalloc("i40e_rss_lut", reta_size, 0);
3535         if (!lut) {
3536                 PMD_DRV_LOG(ERR, "No memory can be allocated");
3537                 return -ENOMEM;
3538         }
3539         ret = i40e_get_rss_lut(pf->main_vsi, lut, reta_size);
3540         if (ret)
3541                 goto out;
3542         for (i = 0; i < reta_size; i++) {
3543                 idx = i / RTE_RETA_GROUP_SIZE;
3544                 shift = i % RTE_RETA_GROUP_SIZE;
3545                 if (reta_conf[idx].mask & (1ULL << shift))
3546                         lut[i] = reta_conf[idx].reta[shift];
3547         }
3548         ret = i40e_set_rss_lut(pf->main_vsi, lut, reta_size);
3549
3550 out:
3551         rte_free(lut);
3552
3553         return ret;
3554 }
3555
3556 static int
3557 i40e_dev_rss_reta_query(struct rte_eth_dev *dev,
3558                         struct rte_eth_rss_reta_entry64 *reta_conf,
3559                         uint16_t reta_size)
3560 {
3561         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3562         uint16_t i, lut_size = pf->hash_lut_size;
3563         uint16_t idx, shift;
3564         uint8_t *lut;
3565         int ret;
3566
3567         if (reta_size != lut_size ||
3568                 reta_size > ETH_RSS_RETA_SIZE_512) {
3569                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
3570                         "(%d) doesn't match the number hardware can supported "
3571                                         "(%d)\n", reta_size, lut_size);
3572                 return -EINVAL;
3573         }
3574
3575         lut = rte_zmalloc("i40e_rss_lut", reta_size, 0);
3576         if (!lut) {
3577                 PMD_DRV_LOG(ERR, "No memory can be allocated");
3578                 return -ENOMEM;
3579         }
3580
3581         ret = i40e_get_rss_lut(pf->main_vsi, lut, reta_size);
3582         if (ret)
3583                 goto out;
3584         for (i = 0; i < reta_size; i++) {
3585                 idx = i / RTE_RETA_GROUP_SIZE;
3586                 shift = i % RTE_RETA_GROUP_SIZE;
3587                 if (reta_conf[idx].mask & (1ULL << shift))
3588                         reta_conf[idx].reta[shift] = lut[i];
3589         }
3590
3591 out:
3592         rte_free(lut);
3593
3594         return ret;
3595 }
3596
3597 /**
3598  * i40e_allocate_dma_mem_d - specific memory alloc for shared code (base driver)
3599  * @hw:   pointer to the HW structure
3600  * @mem:  pointer to mem struct to fill out
3601  * @size: size of memory requested
3602  * @alignment: what to align the allocation to
3603  **/
3604 enum i40e_status_code
3605 i40e_allocate_dma_mem_d(__attribute__((unused)) struct i40e_hw *hw,
3606                         struct i40e_dma_mem *mem,
3607                         u64 size,
3608                         u32 alignment)
3609 {
3610         const struct rte_memzone *mz = NULL;
3611         char z_name[RTE_MEMZONE_NAMESIZE];
3612
3613         if (!mem)
3614                 return I40E_ERR_PARAM;
3615
3616         snprintf(z_name, sizeof(z_name), "i40e_dma_%"PRIu64, rte_rand());
3617         mz = rte_memzone_reserve_bounded(z_name, size, SOCKET_ID_ANY, 0,
3618                                          alignment, RTE_PGSIZE_2M);
3619         if (!mz)
3620                 return I40E_ERR_NO_MEMORY;
3621
3622         mem->size = size;
3623         mem->va = mz->addr;
3624         mem->pa = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
3625         mem->zone = (const void *)mz;
3626         PMD_DRV_LOG(DEBUG, "memzone %s allocated with physical address: "
3627                     "%"PRIu64, mz->name, mem->pa);
3628
3629         return I40E_SUCCESS;
3630 }
3631
3632 /**
3633  * i40e_free_dma_mem_d - specific memory free for shared code (base driver)
3634  * @hw:   pointer to the HW structure
3635  * @mem:  ptr to mem struct to free
3636  **/
3637 enum i40e_status_code
3638 i40e_free_dma_mem_d(__attribute__((unused)) struct i40e_hw *hw,
3639                     struct i40e_dma_mem *mem)
3640 {
3641         if (!mem)
3642                 return I40E_ERR_PARAM;
3643
3644         PMD_DRV_LOG(DEBUG, "memzone %s to be freed with physical address: "
3645                     "%"PRIu64, ((const struct rte_memzone *)mem->zone)->name,
3646                     mem->pa);
3647         rte_memzone_free((const struct rte_memzone *)mem->zone);
3648         mem->zone = NULL;
3649         mem->va = NULL;
3650         mem->pa = (u64)0;
3651
3652         return I40E_SUCCESS;
3653 }
3654
3655 /**
3656  * i40e_allocate_virt_mem_d - specific memory alloc for shared code (base driver)
3657  * @hw:   pointer to the HW structure
3658  * @mem:  pointer to mem struct to fill out
3659  * @size: size of memory requested
3660  **/
3661 enum i40e_status_code
3662 i40e_allocate_virt_mem_d(__attribute__((unused)) struct i40e_hw *hw,
3663                          struct i40e_virt_mem *mem,
3664                          u32 size)
3665 {
3666         if (!mem)
3667                 return I40E_ERR_PARAM;
3668
3669         mem->size = size;
3670         mem->va = rte_zmalloc("i40e", size, 0);
3671
3672         if (mem->va)
3673                 return I40E_SUCCESS;
3674         else
3675                 return I40E_ERR_NO_MEMORY;
3676 }
3677
3678 /**
3679  * i40e_free_virt_mem_d - specific memory free for shared code (base driver)
3680  * @hw:   pointer to the HW structure
3681  * @mem:  pointer to mem struct to free
3682  **/
3683 enum i40e_status_code
3684 i40e_free_virt_mem_d(__attribute__((unused)) struct i40e_hw *hw,
3685                      struct i40e_virt_mem *mem)
3686 {
3687         if (!mem)
3688                 return I40E_ERR_PARAM;
3689
3690         rte_free(mem->va);
3691         mem->va = NULL;
3692
3693         return I40E_SUCCESS;
3694 }
3695
3696 void
3697 i40e_init_spinlock_d(struct i40e_spinlock *sp)
3698 {
3699         rte_spinlock_init(&sp->spinlock);
3700 }
3701
3702 void
3703 i40e_acquire_spinlock_d(struct i40e_spinlock *sp)
3704 {
3705         rte_spinlock_lock(&sp->spinlock);
3706 }
3707
3708 void
3709 i40e_release_spinlock_d(struct i40e_spinlock *sp)
3710 {
3711         rte_spinlock_unlock(&sp->spinlock);
3712 }
3713
3714 void
3715 i40e_destroy_spinlock_d(__attribute__((unused)) struct i40e_spinlock *sp)
3716 {
3717         return;
3718 }
3719
3720 /**
3721  * Get the hardware capabilities, which will be parsed
3722  * and saved into struct i40e_hw.
3723  */
3724 static int
3725 i40e_get_cap(struct i40e_hw *hw)
3726 {
3727         struct i40e_aqc_list_capabilities_element_resp *buf;
3728         uint16_t len, size = 0;
3729         int ret;
3730
3731         /* Calculate a huge enough buff for saving response data temporarily */
3732         len = sizeof(struct i40e_aqc_list_capabilities_element_resp) *
3733                                                 I40E_MAX_CAP_ELE_NUM;
3734         buf = rte_zmalloc("i40e", len, 0);
3735         if (!buf) {
3736                 PMD_DRV_LOG(ERR, "Failed to allocate memory");
3737                 return I40E_ERR_NO_MEMORY;
3738         }
3739
3740         /* Get, parse the capabilities and save it to hw */
3741         ret = i40e_aq_discover_capabilities(hw, buf, len, &size,
3742                         i40e_aqc_opc_list_func_capabilities, NULL);
3743         if (ret != I40E_SUCCESS)
3744                 PMD_DRV_LOG(ERR, "Failed to discover capabilities");
3745
3746         /* Free the temporary buffer after being used */
3747         rte_free(buf);
3748
3749         return ret;
3750 }
3751
3752 static int
3753 i40e_pf_parameter_init(struct rte_eth_dev *dev)
3754 {
3755         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3756         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
3757         uint16_t qp_count = 0, vsi_count = 0;
3758
3759         if (dev->pci_dev->max_vfs && !hw->func_caps.sr_iov_1_1) {
3760                 PMD_INIT_LOG(ERR, "HW configuration doesn't support SRIOV");
3761                 return -EINVAL;
3762         }
3763         /* Add the parameter init for LFC */
3764         pf->fc_conf.pause_time = I40E_DEFAULT_PAUSE_TIME;
3765         pf->fc_conf.high_water[I40E_MAX_TRAFFIC_CLASS] = I40E_DEFAULT_HIGH_WATER;
3766         pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS] = I40E_DEFAULT_LOW_WATER;
3767
3768         pf->flags = I40E_FLAG_HEADER_SPLIT_DISABLED;
3769         pf->max_num_vsi = hw->func_caps.num_vsis;
3770         pf->lan_nb_qp_max = RTE_LIBRTE_I40E_QUEUE_NUM_PER_PF;
3771         pf->vmdq_nb_qp_max = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM;
3772         pf->vf_nb_qp_max = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VF;
3773
3774         /* FDir queue/VSI allocation */
3775         pf->fdir_qp_offset = 0;
3776         if (hw->func_caps.fd) {
3777                 pf->flags |= I40E_FLAG_FDIR;
3778                 pf->fdir_nb_qps = I40E_DEFAULT_QP_NUM_FDIR;
3779         } else {
3780                 pf->fdir_nb_qps = 0;
3781         }
3782         qp_count += pf->fdir_nb_qps;
3783         vsi_count += 1;
3784
3785         /* LAN queue/VSI allocation */
3786         pf->lan_qp_offset = pf->fdir_qp_offset + pf->fdir_nb_qps;
3787         if (!hw->func_caps.rss) {
3788                 pf->lan_nb_qps = 1;
3789         } else {
3790                 pf->flags |= I40E_FLAG_RSS;
3791                 if (hw->mac.type == I40E_MAC_X722)
3792                         pf->flags |= I40E_FLAG_RSS_AQ_CAPABLE;
3793                 pf->lan_nb_qps = pf->lan_nb_qp_max;
3794         }
3795         qp_count += pf->lan_nb_qps;
3796         vsi_count += 1;
3797
3798         /* VF queue/VSI allocation */
3799         pf->vf_qp_offset = pf->lan_qp_offset + pf->lan_nb_qps;
3800         if (hw->func_caps.sr_iov_1_1 && dev->pci_dev->max_vfs) {
3801                 pf->flags |= I40E_FLAG_SRIOV;
3802                 pf->vf_nb_qps = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VF;
3803                 pf->vf_num = dev->pci_dev->max_vfs;
3804                 PMD_DRV_LOG(DEBUG, "%u VF VSIs, %u queues per VF VSI, "
3805                             "in total %u queues", pf->vf_num, pf->vf_nb_qps,
3806                             pf->vf_nb_qps * pf->vf_num);
3807         } else {
3808                 pf->vf_nb_qps = 0;
3809                 pf->vf_num = 0;
3810         }
3811         qp_count += pf->vf_nb_qps * pf->vf_num;
3812         vsi_count += pf->vf_num;
3813
3814         /* VMDq queue/VSI allocation */
3815         pf->vmdq_qp_offset = pf->vf_qp_offset + pf->vf_nb_qps * pf->vf_num;
3816         pf->vmdq_nb_qps = 0;
3817         pf->max_nb_vmdq_vsi = 0;
3818         if (hw->func_caps.vmdq) {
3819                 if (qp_count < hw->func_caps.num_tx_qp &&
3820                         vsi_count < hw->func_caps.num_vsis) {
3821                         pf->max_nb_vmdq_vsi = (hw->func_caps.num_tx_qp -
3822                                 qp_count) / pf->vmdq_nb_qp_max;
3823
3824                         /* Limit the maximum number of VMDq vsi to the maximum
3825                          * ethdev can support
3826                          */
3827                         pf->max_nb_vmdq_vsi = RTE_MIN(pf->max_nb_vmdq_vsi,
3828                                 hw->func_caps.num_vsis - vsi_count);
3829                         pf->max_nb_vmdq_vsi = RTE_MIN(pf->max_nb_vmdq_vsi,
3830                                 ETH_64_POOLS);
3831                         if (pf->max_nb_vmdq_vsi) {
3832                                 pf->flags |= I40E_FLAG_VMDQ;
3833                                 pf->vmdq_nb_qps = pf->vmdq_nb_qp_max;
3834                                 PMD_DRV_LOG(DEBUG, "%u VMDQ VSIs, %u queues "
3835                                             "per VMDQ VSI, in total %u queues",
3836                                             pf->max_nb_vmdq_vsi,
3837                                             pf->vmdq_nb_qps, pf->vmdq_nb_qps *
3838                                             pf->max_nb_vmdq_vsi);
3839                         } else {
3840                                 PMD_DRV_LOG(INFO, "No enough queues left for "
3841                                             "VMDq");
3842                         }
3843                 } else {
3844                         PMD_DRV_LOG(INFO, "No queue or VSI left for VMDq");
3845                 }
3846         }
3847         qp_count += pf->vmdq_nb_qps * pf->max_nb_vmdq_vsi;
3848         vsi_count += pf->max_nb_vmdq_vsi;
3849
3850         if (hw->func_caps.dcb)
3851                 pf->flags |= I40E_FLAG_DCB;
3852
3853         if (qp_count > hw->func_caps.num_tx_qp) {
3854                 PMD_DRV_LOG(ERR, "Failed to allocate %u queues, which exceeds "
3855                             "the hardware maximum %u", qp_count,
3856                             hw->func_caps.num_tx_qp);
3857                 return -EINVAL;
3858         }
3859         if (vsi_count > hw->func_caps.num_vsis) {
3860                 PMD_DRV_LOG(ERR, "Failed to allocate %u VSIs, which exceeds "
3861                             "the hardware maximum %u", vsi_count,
3862                             hw->func_caps.num_vsis);
3863                 return -EINVAL;
3864         }
3865
3866         return 0;
3867 }
3868
3869 static int
3870 i40e_pf_get_switch_config(struct i40e_pf *pf)
3871 {
3872         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
3873         struct i40e_aqc_get_switch_config_resp *switch_config;
3874         struct i40e_aqc_switch_config_element_resp *element;
3875         uint16_t start_seid = 0, num_reported;
3876         int ret;
3877
3878         switch_config = (struct i40e_aqc_get_switch_config_resp *)\
3879                         rte_zmalloc("i40e", I40E_AQ_LARGE_BUF, 0);
3880         if (!switch_config) {
3881                 PMD_DRV_LOG(ERR, "Failed to allocated memory");
3882                 return -ENOMEM;
3883         }
3884
3885         /* Get the switch configurations */
3886         ret = i40e_aq_get_switch_config(hw, switch_config,
3887                 I40E_AQ_LARGE_BUF, &start_seid, NULL);
3888         if (ret != I40E_SUCCESS) {
3889                 PMD_DRV_LOG(ERR, "Failed to get switch configurations");
3890                 goto fail;
3891         }
3892         num_reported = rte_le_to_cpu_16(switch_config->header.num_reported);
3893         if (num_reported != 1) { /* The number should be 1 */
3894                 PMD_DRV_LOG(ERR, "Wrong number of switch config reported");
3895                 goto fail;
3896         }
3897
3898         /* Parse the switch configuration elements */
3899         element = &(switch_config->element[0]);
3900         if (element->element_type == I40E_SWITCH_ELEMENT_TYPE_VSI) {
3901                 pf->mac_seid = rte_le_to_cpu_16(element->uplink_seid);
3902                 pf->main_vsi_seid = rte_le_to_cpu_16(element->seid);
3903         } else
3904                 PMD_DRV_LOG(INFO, "Unknown element type");
3905
3906 fail:
3907         rte_free(switch_config);
3908
3909         return ret;
3910 }
3911
3912 static int
3913 i40e_res_pool_init (struct i40e_res_pool_info *pool, uint32_t base,
3914                         uint32_t num)
3915 {
3916         struct pool_entry *entry;
3917
3918         if (pool == NULL || num == 0)
3919                 return -EINVAL;
3920
3921         entry = rte_zmalloc("i40e", sizeof(*entry), 0);
3922         if (entry == NULL) {
3923                 PMD_DRV_LOG(ERR, "Failed to allocate memory for resource pool");
3924                 return -ENOMEM;
3925         }
3926
3927         /* queue heap initialize */
3928         pool->num_free = num;
3929         pool->num_alloc = 0;
3930         pool->base = base;
3931         LIST_INIT(&pool->alloc_list);
3932         LIST_INIT(&pool->free_list);
3933
3934         /* Initialize element  */
3935         entry->base = 0;
3936         entry->len = num;
3937
3938         LIST_INSERT_HEAD(&pool->free_list, entry, next);
3939         return 0;
3940 }
3941
3942 static void
3943 i40e_res_pool_destroy(struct i40e_res_pool_info *pool)
3944 {
3945         struct pool_entry *entry, *next_entry;
3946
3947         if (pool == NULL)
3948                 return;
3949
3950         for (entry = LIST_FIRST(&pool->alloc_list);
3951                         entry && (next_entry = LIST_NEXT(entry, next), 1);
3952                         entry = next_entry) {
3953                 LIST_REMOVE(entry, next);
3954                 rte_free(entry);
3955         }
3956
3957         for (entry = LIST_FIRST(&pool->free_list);
3958                         entry && (next_entry = LIST_NEXT(entry, next), 1);
3959                         entry = next_entry) {
3960                 LIST_REMOVE(entry, next);
3961                 rte_free(entry);
3962         }
3963
3964         pool->num_free = 0;
3965         pool->num_alloc = 0;
3966         pool->base = 0;
3967         LIST_INIT(&pool->alloc_list);
3968         LIST_INIT(&pool->free_list);
3969 }
3970
3971 static int
3972 i40e_res_pool_free(struct i40e_res_pool_info *pool,
3973                        uint32_t base)
3974 {
3975         struct pool_entry *entry, *next, *prev, *valid_entry = NULL;
3976         uint32_t pool_offset;
3977         int insert;
3978
3979         if (pool == NULL) {
3980                 PMD_DRV_LOG(ERR, "Invalid parameter");
3981                 return -EINVAL;
3982         }
3983
3984         pool_offset = base - pool->base;
3985         /* Lookup in alloc list */
3986         LIST_FOREACH(entry, &pool->alloc_list, next) {
3987                 if (entry->base == pool_offset) {
3988                         valid_entry = entry;
3989                         LIST_REMOVE(entry, next);
3990                         break;
3991                 }
3992         }
3993
3994         /* Not find, return */
3995         if (valid_entry == NULL) {
3996                 PMD_DRV_LOG(ERR, "Failed to find entry");
3997                 return -EINVAL;
3998         }
3999
4000         /**
4001          * Found it, move it to free list  and try to merge.
4002          * In order to make merge easier, always sort it by qbase.
4003          * Find adjacent prev and last entries.
4004          */
4005         prev = next = NULL;
4006         LIST_FOREACH(entry, &pool->free_list, next) {
4007                 if (entry->base > valid_entry->base) {
4008                         next = entry;
4009                         break;
4010                 }
4011                 prev = entry;
4012         }
4013
4014         insert = 0;
4015         /* Try to merge with next one*/
4016         if (next != NULL) {
4017                 /* Merge with next one */
4018                 if (valid_entry->base + valid_entry->len == next->base) {
4019                         next->base = valid_entry->base;
4020                         next->len += valid_entry->len;
4021                         rte_free(valid_entry);
4022                         valid_entry = next;
4023                         insert = 1;
4024                 }
4025         }
4026
4027         if (prev != NULL) {
4028                 /* Merge with previous one */
4029                 if (prev->base + prev->len == valid_entry->base) {
4030                         prev->len += valid_entry->len;
4031                         /* If it merge with next one, remove next node */
4032                         if (insert == 1) {
4033                                 LIST_REMOVE(valid_entry, next);
4034                                 rte_free(valid_entry);
4035                         } else {
4036                                 rte_free(valid_entry);
4037                                 insert = 1;
4038                         }
4039                 }
4040         }
4041
4042         /* Not find any entry to merge, insert */
4043         if (insert == 0) {
4044                 if (prev != NULL)
4045                         LIST_INSERT_AFTER(prev, valid_entry, next);
4046                 else if (next != NULL)
4047                         LIST_INSERT_BEFORE(next, valid_entry, next);
4048                 else /* It's empty list, insert to head */
4049                         LIST_INSERT_HEAD(&pool->free_list, valid_entry, next);
4050         }
4051
4052         pool->num_free += valid_entry->len;
4053         pool->num_alloc -= valid_entry->len;
4054
4055         return 0;
4056 }
4057
4058 static int
4059 i40e_res_pool_alloc(struct i40e_res_pool_info *pool,
4060                        uint16_t num)
4061 {
4062         struct pool_entry *entry, *valid_entry;
4063
4064         if (pool == NULL || num == 0) {
4065                 PMD_DRV_LOG(ERR, "Invalid parameter");
4066                 return -EINVAL;
4067         }
4068
4069         if (pool->num_free < num) {
4070                 PMD_DRV_LOG(ERR, "No resource. ask:%u, available:%u",
4071                             num, pool->num_free);
4072                 return -ENOMEM;
4073         }
4074
4075         valid_entry = NULL;
4076         /* Lookup  in free list and find most fit one */
4077         LIST_FOREACH(entry, &pool->free_list, next) {
4078                 if (entry->len >= num) {
4079                         /* Find best one */
4080                         if (entry->len == num) {
4081                                 valid_entry = entry;
4082                                 break;
4083                         }
4084                         if (valid_entry == NULL || valid_entry->len > entry->len)
4085                                 valid_entry = entry;
4086                 }
4087         }
4088
4089         /* Not find one to satisfy the request, return */
4090         if (valid_entry == NULL) {
4091                 PMD_DRV_LOG(ERR, "No valid entry found");
4092                 return -ENOMEM;
4093         }
4094         /**
4095          * The entry have equal queue number as requested,
4096          * remove it from alloc_list.
4097          */
4098         if (valid_entry->len == num) {
4099                 LIST_REMOVE(valid_entry, next);
4100         } else {
4101                 /**
4102                  * The entry have more numbers than requested,
4103                  * create a new entry for alloc_list and minus its
4104                  * queue base and number in free_list.
4105                  */
4106                 entry = rte_zmalloc("res_pool", sizeof(*entry), 0);
4107                 if (entry == NULL) {
4108                         PMD_DRV_LOG(ERR, "Failed to allocate memory for "
4109                                     "resource pool");
4110                         return -ENOMEM;
4111                 }
4112                 entry->base = valid_entry->base;
4113                 entry->len = num;
4114                 valid_entry->base += num;
4115                 valid_entry->len -= num;
4116                 valid_entry = entry;
4117         }
4118
4119         /* Insert it into alloc list, not sorted */
4120         LIST_INSERT_HEAD(&pool->alloc_list, valid_entry, next);
4121
4122         pool->num_free -= valid_entry->len;
4123         pool->num_alloc += valid_entry->len;
4124
4125         return valid_entry->base + pool->base;
4126 }
4127
4128 /**
4129  * bitmap_is_subset - Check whether src2 is subset of src1
4130  **/
4131 static inline int
4132 bitmap_is_subset(uint8_t src1, uint8_t src2)
4133 {
4134         return !((src1 ^ src2) & src2);
4135 }
4136
4137 static enum i40e_status_code
4138 validate_tcmap_parameter(struct i40e_vsi *vsi, uint8_t enabled_tcmap)
4139 {
4140         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
4141
4142         /* If DCB is not supported, only default TC is supported */
4143         if (!hw->func_caps.dcb && enabled_tcmap != I40E_DEFAULT_TCMAP) {
4144                 PMD_DRV_LOG(ERR, "DCB is not enabled, only TC0 is supported");
4145                 return I40E_NOT_SUPPORTED;
4146         }
4147
4148         if (!bitmap_is_subset(hw->func_caps.enabled_tcmap, enabled_tcmap)) {
4149                 PMD_DRV_LOG(ERR, "Enabled TC map 0x%x not applicable to "
4150                             "HW support 0x%x", hw->func_caps.enabled_tcmap,
4151                             enabled_tcmap);
4152                 return I40E_NOT_SUPPORTED;
4153         }
4154         return I40E_SUCCESS;
4155 }
4156
4157 int
4158 i40e_vsi_vlan_pvid_set(struct i40e_vsi *vsi,
4159                                 struct i40e_vsi_vlan_pvid_info *info)
4160 {
4161         struct i40e_hw *hw;
4162         struct i40e_vsi_context ctxt;
4163         uint8_t vlan_flags = 0;
4164         int ret;
4165
4166         if (vsi == NULL || info == NULL) {
4167                 PMD_DRV_LOG(ERR, "invalid parameters");
4168                 return I40E_ERR_PARAM;
4169         }
4170
4171         if (info->on) {
4172                 vsi->info.pvid = info->config.pvid;
4173                 /**
4174                  * If insert pvid is enabled, only tagged pkts are
4175                  * allowed to be sent out.
4176                  */
4177                 vlan_flags |= I40E_AQ_VSI_PVLAN_INSERT_PVID |
4178                                 I40E_AQ_VSI_PVLAN_MODE_TAGGED;
4179         } else {
4180                 vsi->info.pvid = 0;
4181                 if (info->config.reject.tagged == 0)
4182                         vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_TAGGED;
4183
4184                 if (info->config.reject.untagged == 0)
4185                         vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_UNTAGGED;
4186         }
4187         vsi->info.port_vlan_flags &= ~(I40E_AQ_VSI_PVLAN_INSERT_PVID |
4188                                         I40E_AQ_VSI_PVLAN_MODE_MASK);
4189         vsi->info.port_vlan_flags |= vlan_flags;
4190         vsi->info.valid_sections =
4191                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
4192         memset(&ctxt, 0, sizeof(ctxt));
4193         (void)rte_memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
4194         ctxt.seid = vsi->seid;
4195
4196         hw = I40E_VSI_TO_HW(vsi);
4197         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
4198         if (ret != I40E_SUCCESS)
4199                 PMD_DRV_LOG(ERR, "Failed to update VSI params");
4200
4201         return ret;
4202 }
4203
4204 static int
4205 i40e_vsi_update_tc_bandwidth(struct i40e_vsi *vsi, uint8_t enabled_tcmap)
4206 {
4207         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
4208         int i, ret;
4209         struct i40e_aqc_configure_vsi_tc_bw_data tc_bw_data;
4210
4211         ret = validate_tcmap_parameter(vsi, enabled_tcmap);
4212         if (ret != I40E_SUCCESS)
4213                 return ret;
4214
4215         if (!vsi->seid) {
4216                 PMD_DRV_LOG(ERR, "seid not valid");
4217                 return -EINVAL;
4218         }
4219
4220         memset(&tc_bw_data, 0, sizeof(tc_bw_data));
4221         tc_bw_data.tc_valid_bits = enabled_tcmap;
4222         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
4223                 tc_bw_data.tc_bw_credits[i] =
4224                         (enabled_tcmap & (1 << i)) ? 1 : 0;
4225
4226         ret = i40e_aq_config_vsi_tc_bw(hw, vsi->seid, &tc_bw_data, NULL);
4227         if (ret != I40E_SUCCESS) {
4228                 PMD_DRV_LOG(ERR, "Failed to configure TC BW");
4229                 return ret;
4230         }
4231
4232         (void)rte_memcpy(vsi->info.qs_handle, tc_bw_data.qs_handles,
4233                                         sizeof(vsi->info.qs_handle));
4234         return I40E_SUCCESS;
4235 }
4236
4237 static enum i40e_status_code
4238 i40e_vsi_config_tc_queue_mapping(struct i40e_vsi *vsi,
4239                                  struct i40e_aqc_vsi_properties_data *info,
4240                                  uint8_t enabled_tcmap)
4241 {
4242         enum i40e_status_code ret;
4243         int i, total_tc = 0;
4244         uint16_t qpnum_per_tc, bsf, qp_idx;
4245
4246         ret = validate_tcmap_parameter(vsi, enabled_tcmap);
4247         if (ret != I40E_SUCCESS)
4248                 return ret;
4249
4250         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
4251                 if (enabled_tcmap & (1 << i))
4252                         total_tc++;
4253         if (total_tc == 0)
4254                 total_tc = 1;
4255         vsi->enabled_tc = enabled_tcmap;
4256
4257         /* Number of queues per enabled TC */
4258         qpnum_per_tc = i40e_align_floor(vsi->nb_qps / total_tc);
4259         qpnum_per_tc = RTE_MIN(qpnum_per_tc, I40E_MAX_Q_PER_TC);
4260         bsf = rte_bsf32(qpnum_per_tc);
4261
4262         /* Adjust the queue number to actual queues that can be applied */
4263         if (!(vsi->type == I40E_VSI_MAIN && total_tc == 1))
4264                 vsi->nb_qps = qpnum_per_tc * total_tc;
4265
4266         /**
4267          * Configure TC and queue mapping parameters, for enabled TC,
4268          * allocate qpnum_per_tc queues to this traffic. For disabled TC,
4269          * default queue will serve it.
4270          */
4271         qp_idx = 0;
4272         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4273                 if (vsi->enabled_tc & (1 << i)) {
4274                         info->tc_mapping[i] = rte_cpu_to_le_16((qp_idx <<
4275                                         I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
4276                                 (bsf << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT));
4277                         qp_idx += qpnum_per_tc;
4278                 } else
4279                         info->tc_mapping[i] = 0;
4280         }
4281
4282         /* Associate queue number with VSI */
4283         if (vsi->type == I40E_VSI_SRIOV) {
4284                 info->mapping_flags |=
4285                         rte_cpu_to_le_16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
4286                 for (i = 0; i < vsi->nb_qps; i++)
4287                         info->queue_mapping[i] =
4288                                 rte_cpu_to_le_16(vsi->base_queue + i);
4289         } else {
4290                 info->mapping_flags |=
4291                         rte_cpu_to_le_16(I40E_AQ_VSI_QUE_MAP_CONTIG);
4292                 info->queue_mapping[0] = rte_cpu_to_le_16(vsi->base_queue);
4293         }
4294         info->valid_sections |=
4295                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID);
4296
4297         return I40E_SUCCESS;
4298 }
4299
4300 static int
4301 i40e_veb_release(struct i40e_veb *veb)
4302 {
4303         struct i40e_vsi *vsi;
4304         struct i40e_hw *hw;
4305
4306         if (veb == NULL)
4307                 return -EINVAL;
4308
4309         if (!TAILQ_EMPTY(&veb->head)) {
4310                 PMD_DRV_LOG(ERR, "VEB still has VSI attached, can't remove");
4311                 return -EACCES;
4312         }
4313         /* associate_vsi field is NULL for floating VEB */
4314         if (veb->associate_vsi != NULL) {
4315                 vsi = veb->associate_vsi;
4316                 hw = I40E_VSI_TO_HW(vsi);
4317
4318                 vsi->uplink_seid = veb->uplink_seid;
4319                 vsi->veb = NULL;
4320         } else {
4321                 veb->associate_pf->main_vsi->floating_veb = NULL;
4322                 hw = I40E_VSI_TO_HW(veb->associate_pf->main_vsi);
4323         }
4324
4325         i40e_aq_delete_element(hw, veb->seid, NULL);
4326         rte_free(veb);
4327         return I40E_SUCCESS;
4328 }
4329
4330 /* Setup a veb */
4331 static struct i40e_veb *
4332 i40e_veb_setup(struct i40e_pf *pf, struct i40e_vsi *vsi)
4333 {
4334         struct i40e_veb *veb;
4335         int ret;
4336         struct i40e_hw *hw;
4337
4338         if (pf == NULL) {
4339                 PMD_DRV_LOG(ERR,
4340                             "veb setup failed, associated PF shouldn't null");
4341                 return NULL;
4342         }
4343         hw = I40E_PF_TO_HW(pf);
4344
4345         veb = rte_zmalloc("i40e_veb", sizeof(struct i40e_veb), 0);
4346         if (!veb) {
4347                 PMD_DRV_LOG(ERR, "Failed to allocate memory for veb");
4348                 goto fail;
4349         }
4350
4351         veb->associate_vsi = vsi;
4352         veb->associate_pf = pf;
4353         TAILQ_INIT(&veb->head);
4354         veb->uplink_seid = vsi ? vsi->uplink_seid : 0;
4355
4356         /* create floating veb if vsi is NULL */
4357         if (vsi != NULL) {
4358                 ret = i40e_aq_add_veb(hw, veb->uplink_seid, vsi->seid,
4359                                       I40E_DEFAULT_TCMAP, false,
4360                                       &veb->seid, false, NULL);
4361         } else {
4362                 ret = i40e_aq_add_veb(hw, 0, 0, I40E_DEFAULT_TCMAP,
4363                                       true, &veb->seid, false, NULL);
4364         }
4365
4366         if (ret != I40E_SUCCESS) {
4367                 PMD_DRV_LOG(ERR, "Add veb failed, aq_err: %d",
4368                             hw->aq.asq_last_status);
4369                 goto fail;
4370         }
4371         veb->enabled_tc = I40E_DEFAULT_TCMAP;
4372
4373         /* get statistics index */
4374         ret = i40e_aq_get_veb_parameters(hw, veb->seid, NULL, NULL,
4375                                 &veb->stats_idx, NULL, NULL, NULL);
4376         if (ret != I40E_SUCCESS) {
4377                 PMD_DRV_LOG(ERR, "Get veb statics index failed, aq_err: %d",
4378                             hw->aq.asq_last_status);
4379                 goto fail;
4380         }
4381         /* Get VEB bandwidth, to be implemented */
4382         /* Now associated vsi binding to the VEB, set uplink to this VEB */
4383         if (vsi)
4384                 vsi->uplink_seid = veb->seid;
4385
4386         return veb;
4387 fail:
4388         rte_free(veb);
4389         return NULL;
4390 }
4391
4392 int
4393 i40e_vsi_release(struct i40e_vsi *vsi)
4394 {
4395         struct i40e_pf *pf;
4396         struct i40e_hw *hw;
4397         struct i40e_vsi_list *vsi_list;
4398         void *temp;
4399         int ret;
4400         struct i40e_mac_filter *f;
4401         uint16_t user_param;
4402
4403         if (!vsi)
4404                 return I40E_SUCCESS;
4405
4406         if (!vsi->adapter)
4407                 return -EFAULT;
4408
4409         user_param = vsi->user_param;
4410
4411         pf = I40E_VSI_TO_PF(vsi);
4412         hw = I40E_VSI_TO_HW(vsi);
4413
4414         /* VSI has child to attach, release child first */
4415         if (vsi->veb) {
4416                 TAILQ_FOREACH_SAFE(vsi_list, &vsi->veb->head, list, temp) {
4417                         if (i40e_vsi_release(vsi_list->vsi) != I40E_SUCCESS)
4418                                 return -1;
4419                 }
4420                 i40e_veb_release(vsi->veb);
4421         }
4422
4423         if (vsi->floating_veb) {
4424                 TAILQ_FOREACH_SAFE(vsi_list, &vsi->floating_veb->head, list, temp) {
4425                         if (i40e_vsi_release(vsi_list->vsi) != I40E_SUCCESS)
4426                                 return -1;
4427                 }
4428         }
4429
4430         /* Remove all macvlan filters of the VSI */
4431         i40e_vsi_remove_all_macvlan_filter(vsi);
4432         TAILQ_FOREACH_SAFE(f, &vsi->mac_list, next, temp)
4433                 rte_free(f);
4434
4435         if (vsi->type != I40E_VSI_MAIN &&
4436             ((vsi->type != I40E_VSI_SRIOV) ||
4437             !pf->floating_veb_list[user_param])) {
4438                 /* Remove vsi from parent's sibling list */
4439                 if (vsi->parent_vsi == NULL || vsi->parent_vsi->veb == NULL) {
4440                         PMD_DRV_LOG(ERR, "VSI's parent VSI is NULL");
4441                         return I40E_ERR_PARAM;
4442                 }
4443                 TAILQ_REMOVE(&vsi->parent_vsi->veb->head,
4444                                 &vsi->sib_vsi_list, list);
4445
4446                 /* Remove all switch element of the VSI */
4447                 ret = i40e_aq_delete_element(hw, vsi->seid, NULL);
4448                 if (ret != I40E_SUCCESS)
4449                         PMD_DRV_LOG(ERR, "Failed to delete element");
4450         }
4451
4452         if ((vsi->type == I40E_VSI_SRIOV) &&
4453             pf->floating_veb_list[user_param]) {
4454                 /* Remove vsi from parent's sibling list */
4455                 if (vsi->parent_vsi == NULL ||
4456                     vsi->parent_vsi->floating_veb == NULL) {
4457                         PMD_DRV_LOG(ERR, "VSI's parent VSI is NULL");
4458                         return I40E_ERR_PARAM;
4459                 }
4460                 TAILQ_REMOVE(&vsi->parent_vsi->floating_veb->head,
4461                              &vsi->sib_vsi_list, list);
4462
4463                 /* Remove all switch element of the VSI */
4464                 ret = i40e_aq_delete_element(hw, vsi->seid, NULL);
4465                 if (ret != I40E_SUCCESS)
4466                         PMD_DRV_LOG(ERR, "Failed to delete element");
4467         }
4468
4469         i40e_res_pool_free(&pf->qp_pool, vsi->base_queue);
4470
4471         if (vsi->type != I40E_VSI_SRIOV)
4472                 i40e_res_pool_free(&pf->msix_pool, vsi->msix_intr);
4473         rte_free(vsi);
4474
4475         return I40E_SUCCESS;
4476 }
4477
4478 static int
4479 i40e_update_default_filter_setting(struct i40e_vsi *vsi)
4480 {
4481         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
4482         struct i40e_aqc_remove_macvlan_element_data def_filter;
4483         struct i40e_mac_filter_info filter;
4484         int ret;
4485
4486         if (vsi->type != I40E_VSI_MAIN)
4487                 return I40E_ERR_CONFIG;
4488         memset(&def_filter, 0, sizeof(def_filter));
4489         (void)rte_memcpy(def_filter.mac_addr, hw->mac.perm_addr,
4490                                         ETH_ADDR_LEN);
4491         def_filter.vlan_tag = 0;
4492         def_filter.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
4493                                 I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
4494         ret = i40e_aq_remove_macvlan(hw, vsi->seid, &def_filter, 1, NULL);
4495         if (ret != I40E_SUCCESS) {
4496                 struct i40e_mac_filter *f;
4497                 struct ether_addr *mac;
4498
4499                 PMD_DRV_LOG(WARNING, "Cannot remove the default "
4500                             "macvlan filter");
4501                 /* It needs to add the permanent mac into mac list */
4502                 f = rte_zmalloc("macv_filter", sizeof(*f), 0);
4503                 if (f == NULL) {
4504                         PMD_DRV_LOG(ERR, "failed to allocate memory");
4505                         return I40E_ERR_NO_MEMORY;
4506                 }
4507                 mac = &f->mac_info.mac_addr;
4508                 (void)rte_memcpy(&mac->addr_bytes, hw->mac.perm_addr,
4509                                 ETH_ADDR_LEN);
4510                 f->mac_info.filter_type = RTE_MACVLAN_PERFECT_MATCH;
4511                 TAILQ_INSERT_TAIL(&vsi->mac_list, f, next);
4512                 vsi->mac_num++;
4513
4514                 return ret;
4515         }
4516         (void)rte_memcpy(&filter.mac_addr,
4517                 (struct ether_addr *)(hw->mac.perm_addr), ETH_ADDR_LEN);
4518         filter.filter_type = RTE_MACVLAN_PERFECT_MATCH;
4519         return i40e_vsi_add_mac(vsi, &filter);
4520 }
4521
4522 /*
4523  * i40e_vsi_get_bw_config - Query VSI BW Information
4524  * @vsi: the VSI to be queried
4525  *
4526  * Returns 0 on success, negative value on failure
4527  */
4528 static enum i40e_status_code
4529 i40e_vsi_get_bw_config(struct i40e_vsi *vsi)
4530 {
4531         struct i40e_aqc_query_vsi_bw_config_resp bw_config;
4532         struct i40e_aqc_query_vsi_ets_sla_config_resp ets_sla_config;
4533         struct i40e_hw *hw = &vsi->adapter->hw;
4534         i40e_status ret;
4535         int i;
4536         uint32_t bw_max;
4537
4538         memset(&bw_config, 0, sizeof(bw_config));
4539         ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
4540         if (ret != I40E_SUCCESS) {
4541                 PMD_DRV_LOG(ERR, "VSI failed to get bandwidth configuration %u",
4542                             hw->aq.asq_last_status);
4543                 return ret;
4544         }
4545
4546         memset(&ets_sla_config, 0, sizeof(ets_sla_config));
4547         ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid,
4548                                         &ets_sla_config, NULL);
4549         if (ret != I40E_SUCCESS) {
4550                 PMD_DRV_LOG(ERR, "VSI failed to get TC bandwdith "
4551                             "configuration %u", hw->aq.asq_last_status);
4552                 return ret;
4553         }
4554
4555         /* store and print out BW info */
4556         vsi->bw_info.bw_limit = rte_le_to_cpu_16(bw_config.port_bw_limit);
4557         vsi->bw_info.bw_max = bw_config.max_bw;
4558         PMD_DRV_LOG(DEBUG, "VSI bw limit:%u", vsi->bw_info.bw_limit);
4559         PMD_DRV_LOG(DEBUG, "VSI max_bw:%u", vsi->bw_info.bw_max);
4560         bw_max = rte_le_to_cpu_16(ets_sla_config.tc_bw_max[0]) |
4561                     (rte_le_to_cpu_16(ets_sla_config.tc_bw_max[1]) <<
4562                      I40E_16_BIT_WIDTH);
4563         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4564                 vsi->bw_info.bw_ets_share_credits[i] =
4565                                 ets_sla_config.share_credits[i];
4566                 vsi->bw_info.bw_ets_credits[i] =
4567                                 rte_le_to_cpu_16(ets_sla_config.credits[i]);
4568                 /* 4 bits per TC, 4th bit is reserved */
4569                 vsi->bw_info.bw_ets_max[i] =
4570                         (uint8_t)((bw_max >> (i * I40E_4_BIT_WIDTH)) &
4571                                   RTE_LEN2MASK(3, uint8_t));
4572                 PMD_DRV_LOG(DEBUG, "\tVSI TC%u:share credits %u", i,
4573                             vsi->bw_info.bw_ets_share_credits[i]);
4574                 PMD_DRV_LOG(DEBUG, "\tVSI TC%u:credits %u", i,
4575                             vsi->bw_info.bw_ets_credits[i]);
4576                 PMD_DRV_LOG(DEBUG, "\tVSI TC%u: max credits: %u", i,
4577                             vsi->bw_info.bw_ets_max[i]);
4578         }
4579
4580         return I40E_SUCCESS;
4581 }
4582
4583 /* i40e_enable_pf_lb
4584  * @pf: pointer to the pf structure
4585  *
4586  * allow loopback on pf
4587  */
4588 static inline void
4589 i40e_enable_pf_lb(struct i40e_pf *pf)
4590 {
4591         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
4592         struct i40e_vsi_context ctxt;
4593         int ret;
4594
4595         /* Use the FW API if FW >= v5.0 */
4596         if (hw->aq.fw_maj_ver < 5 && hw->mac.type != I40E_MAC_X722) {
4597                 PMD_INIT_LOG(ERR, "FW < v5.0, cannot enable loopback");
4598                 return;
4599         }
4600
4601         memset(&ctxt, 0, sizeof(ctxt));
4602         ctxt.seid = pf->main_vsi_seid;
4603         ctxt.pf_num = hw->pf_id;
4604         ret = i40e_aq_get_vsi_params(hw, &ctxt, NULL);
4605         if (ret) {
4606                 PMD_DRV_LOG(ERR, "cannot get pf vsi config, err %d, aq_err %d",
4607                             ret, hw->aq.asq_last_status);
4608                 return;
4609         }
4610         ctxt.flags = I40E_AQ_VSI_TYPE_PF;
4611         ctxt.info.valid_sections =
4612                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SWITCH_VALID);
4613         ctxt.info.switch_id |=
4614                 rte_cpu_to_le_16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
4615
4616         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
4617         if (ret)
4618                 PMD_DRV_LOG(ERR, "update vsi switch failed, aq_err=%d\n",
4619                             hw->aq.asq_last_status);
4620 }
4621
4622 /* Setup a VSI */
4623 struct i40e_vsi *
4624 i40e_vsi_setup(struct i40e_pf *pf,
4625                enum i40e_vsi_type type,
4626                struct i40e_vsi *uplink_vsi,
4627                uint16_t user_param)
4628 {
4629         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
4630         struct i40e_vsi *vsi;
4631         struct i40e_mac_filter_info filter;
4632         int ret;
4633         struct i40e_vsi_context ctxt;
4634         struct ether_addr broadcast =
4635                 {.addr_bytes = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}};
4636
4637         if (type != I40E_VSI_MAIN && type != I40E_VSI_SRIOV &&
4638             uplink_vsi == NULL) {
4639                 PMD_DRV_LOG(ERR, "VSI setup failed, "
4640                             "VSI link shouldn't be NULL");
4641                 return NULL;
4642         }
4643
4644         if (type == I40E_VSI_MAIN && uplink_vsi != NULL) {
4645                 PMD_DRV_LOG(ERR, "VSI setup failed, MAIN VSI "
4646                             "uplink VSI should be NULL");
4647                 return NULL;
4648         }
4649
4650         /* two situations
4651          * 1.type is not MAIN and uplink vsi is not NULL
4652          * If uplink vsi didn't setup VEB, create one first under veb field
4653          * 2.type is SRIOV and the uplink is NULL
4654          * If floating VEB is NULL, create one veb under floating veb field
4655          */
4656
4657         if (type != I40E_VSI_MAIN && uplink_vsi != NULL &&
4658             uplink_vsi->veb == NULL) {
4659                 uplink_vsi->veb = i40e_veb_setup(pf, uplink_vsi);
4660
4661                 if (uplink_vsi->veb == NULL) {
4662                         PMD_DRV_LOG(ERR, "VEB setup failed");
4663                         return NULL;
4664                 }
4665                 /* set ALLOWLOOPBACk on pf, when veb is created */
4666                 i40e_enable_pf_lb(pf);
4667         }
4668
4669         if (type == I40E_VSI_SRIOV && uplink_vsi == NULL &&
4670             pf->main_vsi->floating_veb == NULL) {
4671                 pf->main_vsi->floating_veb = i40e_veb_setup(pf, uplink_vsi);
4672
4673                 if (pf->main_vsi->floating_veb == NULL) {
4674                         PMD_DRV_LOG(ERR, "VEB setup failed");
4675                         return NULL;
4676                 }
4677         }
4678
4679         vsi = rte_zmalloc("i40e_vsi", sizeof(struct i40e_vsi), 0);
4680         if (!vsi) {
4681                 PMD_DRV_LOG(ERR, "Failed to allocate memory for vsi");
4682                 return NULL;
4683         }
4684         TAILQ_INIT(&vsi->mac_list);
4685         vsi->type = type;
4686         vsi->adapter = I40E_PF_TO_ADAPTER(pf);
4687         vsi->max_macaddrs = I40E_NUM_MACADDR_MAX;
4688         vsi->parent_vsi = uplink_vsi ? uplink_vsi : pf->main_vsi;
4689         vsi->user_param = user_param;
4690         /* Allocate queues */
4691         switch (vsi->type) {
4692         case I40E_VSI_MAIN  :
4693                 vsi->nb_qps = pf->lan_nb_qps;
4694                 break;
4695         case I40E_VSI_SRIOV :
4696                 vsi->nb_qps = pf->vf_nb_qps;
4697                 break;
4698         case I40E_VSI_VMDQ2:
4699                 vsi->nb_qps = pf->vmdq_nb_qps;
4700                 break;
4701         case I40E_VSI_FDIR:
4702                 vsi->nb_qps = pf->fdir_nb_qps;
4703                 break;
4704         default:
4705                 goto fail_mem;
4706         }
4707         /*
4708          * The filter status descriptor is reported in rx queue 0,
4709          * while the tx queue for fdir filter programming has no
4710          * such constraints, can be non-zero queues.
4711          * To simplify it, choose FDIR vsi use queue 0 pair.
4712          * To make sure it will use queue 0 pair, queue allocation
4713          * need be done before this function is called
4714          */
4715         if (type != I40E_VSI_FDIR) {
4716                 ret = i40e_res_pool_alloc(&pf->qp_pool, vsi->nb_qps);
4717                         if (ret < 0) {
4718                                 PMD_DRV_LOG(ERR, "VSI %d allocate queue failed %d",
4719                                                 vsi->seid, ret);
4720                                 goto fail_mem;
4721                         }
4722                         vsi->base_queue = ret;
4723         } else
4724                 vsi->base_queue = I40E_FDIR_QUEUE_ID;
4725
4726         /* VF has MSIX interrupt in VF range, don't allocate here */
4727         if (type == I40E_VSI_MAIN) {
4728                 if (pf->support_multi_driver) {
4729                         /* If support multi-driver, need to use INT0 instead of
4730                          * allocating from msix pool. The Msix pool is init from
4731                          * INT1, so it's OK just set msix_intr to 0 and nb_msix
4732                          * to 1 without calling i40e_res_pool_alloc.
4733                          */
4734                         vsi->msix_intr = 0;
4735                         vsi->nb_msix = 1;
4736                 } else {
4737                         ret = i40e_res_pool_alloc(&pf->msix_pool,
4738                                                   RTE_MIN(vsi->nb_qps,
4739                                                      RTE_MAX_RXTX_INTR_VEC_ID));
4740                         if (ret < 0) {
4741                                 PMD_DRV_LOG(ERR,
4742                                             "VSI MAIN %d get heap failed %d",
4743                                             vsi->seid, ret);
4744                                 goto fail_queue_alloc;
4745                         }
4746                         vsi->msix_intr = ret;
4747                         vsi->nb_msix = RTE_MIN(vsi->nb_qps,
4748                                                RTE_MAX_RXTX_INTR_VEC_ID);
4749                 }
4750         } else if (type != I40E_VSI_SRIOV) {
4751                 ret = i40e_res_pool_alloc(&pf->msix_pool, 1);
4752                 if (ret < 0) {
4753                         PMD_DRV_LOG(ERR, "VSI %d get heap failed %d", vsi->seid, ret);
4754                         goto fail_queue_alloc;
4755                 }
4756                 vsi->msix_intr = ret;
4757                 vsi->nb_msix = 1;
4758         } else {
4759                 vsi->msix_intr = 0;
4760                 vsi->nb_msix = 0;
4761         }
4762
4763         /* Add VSI */
4764         if (type == I40E_VSI_MAIN) {
4765                 /* For main VSI, no need to add since it's default one */
4766                 vsi->uplink_seid = pf->mac_seid;
4767                 vsi->seid = pf->main_vsi_seid;
4768                 /* Bind queues with specific MSIX interrupt */
4769                 /**
4770                  * Needs 2 interrupt at least, one for misc cause which will
4771                  * enabled from OS side, Another for queues binding the
4772                  * interrupt from device side only.
4773                  */
4774
4775                 /* Get default VSI parameters from hardware */
4776                 memset(&ctxt, 0, sizeof(ctxt));
4777                 ctxt.seid = vsi->seid;
4778                 ctxt.pf_num = hw->pf_id;
4779                 ctxt.uplink_seid = vsi->uplink_seid;
4780                 ctxt.vf_num = 0;
4781                 ret = i40e_aq_get_vsi_params(hw, &ctxt, NULL);
4782                 if (ret != I40E_SUCCESS) {
4783                         PMD_DRV_LOG(ERR, "Failed to get VSI params");
4784                         goto fail_msix_alloc;
4785                 }
4786                 (void)rte_memcpy(&vsi->info, &ctxt.info,
4787                         sizeof(struct i40e_aqc_vsi_properties_data));
4788                 vsi->vsi_id = ctxt.vsi_number;
4789                 vsi->info.valid_sections = 0;
4790
4791                 /* Configure tc, enabled TC0 only */
4792                 if (i40e_vsi_update_tc_bandwidth(vsi, I40E_DEFAULT_TCMAP) !=
4793                         I40E_SUCCESS) {
4794                         PMD_DRV_LOG(ERR, "Failed to update TC bandwidth");
4795                         goto fail_msix_alloc;
4796                 }
4797
4798                 /* TC, queue mapping */
4799                 memset(&ctxt, 0, sizeof(ctxt));
4800                 vsi->info.valid_sections |=
4801                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
4802                 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
4803                                         I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
4804                 (void)rte_memcpy(&ctxt.info, &vsi->info,
4805                         sizeof(struct i40e_aqc_vsi_properties_data));
4806                 ret = i40e_vsi_config_tc_queue_mapping(vsi, &ctxt.info,
4807                                                 I40E_DEFAULT_TCMAP);
4808                 if (ret != I40E_SUCCESS) {
4809                         PMD_DRV_LOG(ERR, "Failed to configure "
4810                                     "TC queue mapping");
4811                         goto fail_msix_alloc;
4812                 }
4813                 ctxt.seid = vsi->seid;
4814                 ctxt.pf_num = hw->pf_id;
4815                 ctxt.uplink_seid = vsi->uplink_seid;
4816                 ctxt.vf_num = 0;
4817
4818                 /* Update VSI parameters */
4819                 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
4820                 if (ret != I40E_SUCCESS) {
4821                         PMD_DRV_LOG(ERR, "Failed to update VSI params");
4822                         goto fail_msix_alloc;
4823                 }
4824
4825                 (void)rte_memcpy(&vsi->info.tc_mapping, &ctxt.info.tc_mapping,
4826                                                 sizeof(vsi->info.tc_mapping));
4827                 (void)rte_memcpy(&vsi->info.queue_mapping,
4828                                 &ctxt.info.queue_mapping,
4829                         sizeof(vsi->info.queue_mapping));
4830                 vsi->info.mapping_flags = ctxt.info.mapping_flags;
4831                 vsi->info.valid_sections = 0;
4832
4833                 (void)rte_memcpy(pf->dev_addr.addr_bytes, hw->mac.perm_addr,
4834                                 ETH_ADDR_LEN);
4835
4836                 /**
4837                  * Updating default filter settings are necessary to prevent
4838                  * reception of tagged packets.
4839                  * Some old firmware configurations load a default macvlan
4840                  * filter which accepts both tagged and untagged packets.
4841                  * The updating is to use a normal filter instead if needed.
4842                  * For NVM 4.2.2 or after, the updating is not needed anymore.
4843                  * The firmware with correct configurations load the default
4844                  * macvlan filter which is expected and cannot be removed.
4845                  */
4846                 i40e_update_default_filter_setting(vsi);
4847                 i40e_config_qinq(hw, vsi);
4848         } else if (type == I40E_VSI_SRIOV) {
4849                 memset(&ctxt, 0, sizeof(ctxt));
4850                 /**
4851                  * For other VSI, the uplink_seid equals to uplink VSI's
4852                  * uplink_seid since they share same VEB
4853                  */
4854                 if (uplink_vsi == NULL)
4855                         vsi->uplink_seid = pf->main_vsi->floating_veb->seid;
4856                 else
4857                         vsi->uplink_seid = uplink_vsi->uplink_seid;
4858                 ctxt.pf_num = hw->pf_id;
4859                 ctxt.vf_num = hw->func_caps.vf_base_id + user_param;
4860                 ctxt.uplink_seid = vsi->uplink_seid;
4861                 ctxt.connection_type = 0x1;
4862                 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
4863
4864                 /* Use the VEB configuration if FW >= v5.0 */
4865                 if (hw->aq.fw_maj_ver >= 5 || hw->mac.type == I40E_MAC_X722) {
4866                         /* Configure switch ID */
4867                         ctxt.info.valid_sections |=
4868                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SWITCH_VALID);
4869                         ctxt.info.switch_id =
4870                         rte_cpu_to_le_16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
4871                 }
4872
4873                 /* Configure port/vlan */
4874                 ctxt.info.valid_sections |=
4875                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
4876                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
4877                 ret = i40e_vsi_config_tc_queue_mapping(vsi, &ctxt.info,
4878                                                 I40E_DEFAULT_TCMAP);
4879                 if (ret != I40E_SUCCESS) {
4880                         PMD_DRV_LOG(ERR, "Failed to configure "
4881                                     "TC queue mapping");
4882                         goto fail_msix_alloc;
4883                 }
4884                 ctxt.info.up_enable_bits = I40E_DEFAULT_TCMAP;
4885                 ctxt.info.valid_sections |=
4886                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SCHED_VALID);
4887                 /**
4888                  * Since VSI is not created yet, only configure parameter,
4889                  * will add vsi below.
4890                  */
4891
4892                 i40e_config_qinq(hw, vsi);
4893         } else if (type == I40E_VSI_VMDQ2) {
4894                 memset(&ctxt, 0, sizeof(ctxt));
4895                 /*
4896                  * For other VSI, the uplink_seid equals to uplink VSI's
4897                  * uplink_seid since they share same VEB
4898                  */
4899                 vsi->uplink_seid = uplink_vsi->uplink_seid;
4900                 ctxt.pf_num = hw->pf_id;
4901                 ctxt.vf_num = 0;
4902                 ctxt.uplink_seid = vsi->uplink_seid;
4903                 ctxt.connection_type = 0x1;
4904                 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
4905
4906                 ctxt.info.valid_sections |=
4907                                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SWITCH_VALID);
4908                 /* user_param carries flag to enable loop back */
4909                 if (user_param) {
4910                         ctxt.info.switch_id =
4911                         rte_cpu_to_le_16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB);
4912                         ctxt.info.switch_id |=
4913                         rte_cpu_to_le_16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
4914                 }
4915
4916                 /* Configure port/vlan */
4917                 ctxt.info.valid_sections |=
4918                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
4919                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
4920                 ret = i40e_vsi_config_tc_queue_mapping(vsi, &ctxt.info,
4921                                                 I40E_DEFAULT_TCMAP);
4922                 if (ret != I40E_SUCCESS) {
4923                         PMD_DRV_LOG(ERR, "Failed to configure "
4924                                         "TC queue mapping");
4925                         goto fail_msix_alloc;
4926                 }
4927                 ctxt.info.up_enable_bits = I40E_DEFAULT_TCMAP;
4928                 ctxt.info.valid_sections |=
4929                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SCHED_VALID);
4930         } else if (type == I40E_VSI_FDIR) {
4931                 memset(&ctxt, 0, sizeof(ctxt));
4932                 vsi->uplink_seid = uplink_vsi->uplink_seid;
4933                 ctxt.pf_num = hw->pf_id;
4934                 ctxt.vf_num = 0;
4935                 ctxt.uplink_seid = vsi->uplink_seid;
4936                 ctxt.connection_type = 0x1;     /* regular data port */
4937                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
4938                 ret = i40e_vsi_config_tc_queue_mapping(vsi, &ctxt.info,
4939                                                 I40E_DEFAULT_TCMAP);
4940                 if (ret != I40E_SUCCESS) {
4941                         PMD_DRV_LOG(ERR, "Failed to configure "
4942                                         "TC queue mapping.");
4943                         goto fail_msix_alloc;
4944                 }
4945                 ctxt.info.up_enable_bits = I40E_DEFAULT_TCMAP;
4946                 ctxt.info.valid_sections |=
4947                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SCHED_VALID);
4948         } else {
4949                 PMD_DRV_LOG(ERR, "VSI: Not support other type VSI yet");
4950                 goto fail_msix_alloc;
4951         }
4952
4953         if (vsi->type != I40E_VSI_MAIN) {
4954                 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
4955                 if (ret != I40E_SUCCESS) {
4956                         PMD_DRV_LOG(ERR, "add vsi failed, aq_err=%d",
4957                                     hw->aq.asq_last_status);
4958                         goto fail_msix_alloc;
4959                 }
4960                 memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info));
4961                 vsi->info.valid_sections = 0;
4962                 vsi->seid = ctxt.seid;
4963                 vsi->vsi_id = ctxt.vsi_number;
4964                 vsi->sib_vsi_list.vsi = vsi;
4965                 if (vsi->type == I40E_VSI_SRIOV && uplink_vsi == NULL) {
4966                         TAILQ_INSERT_TAIL(&pf->main_vsi->floating_veb->head,
4967                                           &vsi->sib_vsi_list, list);
4968                 } else {
4969                         TAILQ_INSERT_TAIL(&uplink_vsi->veb->head,
4970                                           &vsi->sib_vsi_list, list);
4971                 }
4972         }
4973
4974         /* MAC/VLAN configuration */
4975         (void)rte_memcpy(&filter.mac_addr, &broadcast, ETHER_ADDR_LEN);
4976         filter.filter_type = RTE_MACVLAN_PERFECT_MATCH;
4977
4978         ret = i40e_vsi_add_mac(vsi, &filter);
4979         if (ret != I40E_SUCCESS) {
4980                 PMD_DRV_LOG(ERR, "Failed to add MACVLAN filter");
4981                 goto fail_msix_alloc;
4982         }
4983
4984         /* Get VSI BW information */
4985         i40e_vsi_get_bw_config(vsi);
4986         return vsi;
4987 fail_msix_alloc:
4988         i40e_res_pool_free(&pf->msix_pool,vsi->msix_intr);
4989 fail_queue_alloc:
4990         i40e_res_pool_free(&pf->qp_pool,vsi->base_queue);
4991 fail_mem:
4992         rte_free(vsi);
4993         return NULL;
4994 }
4995
4996 /* Configure vlan filter on or off */
4997 int
4998 i40e_vsi_config_vlan_filter(struct i40e_vsi *vsi, bool on)
4999 {
5000         int i, num;
5001         struct i40e_mac_filter *f;
5002         void *temp;
5003         struct i40e_mac_filter_info *mac_filter;
5004         enum rte_mac_filter_type desired_filter;
5005         int ret = I40E_SUCCESS;
5006
5007         if (on) {
5008                 /* Filter to match MAC and VLAN */
5009                 desired_filter = RTE_MACVLAN_PERFECT_MATCH;
5010         } else {
5011                 /* Filter to match only MAC */
5012                 desired_filter = RTE_MAC_PERFECT_MATCH;
5013         }
5014
5015         num = vsi->mac_num;
5016
5017         mac_filter = rte_zmalloc("mac_filter_info_data",
5018                                  num * sizeof(*mac_filter), 0);
5019         if (mac_filter == NULL) {
5020                 PMD_DRV_LOG(ERR, "failed to allocate memory");
5021                 return I40E_ERR_NO_MEMORY;
5022         }
5023
5024         i = 0;
5025
5026         /* Remove all existing mac */
5027         TAILQ_FOREACH_SAFE(f, &vsi->mac_list, next, temp) {
5028                 mac_filter[i] = f->mac_info;
5029                 ret = i40e_vsi_delete_mac(vsi, &f->mac_info.mac_addr);
5030                 if (ret) {
5031                         PMD_DRV_LOG(ERR, "Update VSI failed to %s vlan filter",
5032                                     on ? "enable" : "disable");
5033                         goto DONE;
5034                 }
5035                 i++;
5036         }
5037
5038         /* Override with new filter */
5039         for (i = 0; i < num; i++) {
5040                 mac_filter[i].filter_type = desired_filter;
5041                 ret = i40e_vsi_add_mac(vsi, &mac_filter[i]);
5042                 if (ret) {
5043                         PMD_DRV_LOG(ERR, "Update VSI failed to %s vlan filter",
5044                                     on ? "enable" : "disable");
5045                         goto DONE;
5046                 }
5047         }
5048
5049 DONE:
5050         rte_free(mac_filter);
5051         return ret;
5052 }
5053
5054 /* Configure vlan stripping on or off */
5055 int
5056 i40e_vsi_config_vlan_stripping(struct i40e_vsi *vsi, bool on)
5057 {
5058         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
5059         struct i40e_vsi_context ctxt;
5060         uint8_t vlan_flags;
5061         int ret = I40E_SUCCESS;
5062
5063         /* Check if it has been already on or off */
5064         if (vsi->info.valid_sections &
5065                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID)) {
5066                 if (on) {
5067                         if ((vsi->info.port_vlan_flags &
5068                                 I40E_AQ_VSI_PVLAN_EMOD_MASK) == 0)
5069                                 return 0; /* already on */
5070                 } else {
5071                         if ((vsi->info.port_vlan_flags &
5072                                 I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
5073                                 I40E_AQ_VSI_PVLAN_EMOD_MASK)
5074                                 return 0; /* already off */
5075                 }
5076         }
5077
5078         if (on)
5079                 vlan_flags = I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
5080         else
5081                 vlan_flags = I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
5082         vsi->info.valid_sections =
5083                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
5084         vsi->info.port_vlan_flags &= ~(I40E_AQ_VSI_PVLAN_EMOD_MASK);
5085         vsi->info.port_vlan_flags |= vlan_flags;
5086         ctxt.seid = vsi->seid;
5087         (void)rte_memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
5088         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
5089         if (ret)
5090                 PMD_DRV_LOG(INFO, "Update VSI failed to %s vlan stripping",
5091                             on ? "enable" : "disable");
5092
5093         return ret;
5094 }
5095
5096 static int
5097 i40e_dev_init_vlan(struct rte_eth_dev *dev)
5098 {
5099         struct rte_eth_dev_data *data = dev->data;
5100         int ret;
5101         int mask = 0;
5102
5103         /* Apply vlan offload setting */
5104         mask = ETH_VLAN_STRIP_MASK |
5105                ETH_VLAN_FILTER_MASK |
5106                ETH_VLAN_EXTEND_MASK;
5107         i40e_vlan_offload_set(dev, mask);
5108
5109         /* Apply pvid setting */
5110         ret = i40e_vlan_pvid_set(dev, data->dev_conf.txmode.pvid,
5111                                 data->dev_conf.txmode.hw_vlan_insert_pvid);
5112         if (ret)
5113                 PMD_DRV_LOG(INFO, "Failed to update VSI params");
5114
5115         return ret;
5116 }
5117
5118 static int
5119 i40e_vsi_config_double_vlan(struct i40e_vsi *vsi, int on)
5120 {
5121         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
5122
5123         return i40e_aq_set_port_parameters(hw, vsi->seid, 0, 1, on, NULL);
5124 }
5125
5126 static int
5127 i40e_update_flow_control(struct i40e_hw *hw)
5128 {
5129 #define I40E_LINK_PAUSE_RXTX (I40E_AQ_LINK_PAUSE_RX | I40E_AQ_LINK_PAUSE_TX)
5130         struct i40e_link_status link_status;
5131         uint32_t rxfc = 0, txfc = 0, reg;
5132         uint8_t an_info;
5133         int ret;
5134
5135         memset(&link_status, 0, sizeof(link_status));
5136         ret = i40e_aq_get_link_info(hw, FALSE, &link_status, NULL);
5137         if (ret != I40E_SUCCESS) {
5138                 PMD_DRV_LOG(ERR, "Failed to get link status information");
5139                 goto write_reg; /* Disable flow control */
5140         }
5141
5142         an_info = hw->phy.link_info.an_info;
5143         if (!(an_info & I40E_AQ_AN_COMPLETED)) {
5144                 PMD_DRV_LOG(INFO, "Link auto negotiation not completed");
5145                 ret = I40E_ERR_NOT_READY;
5146                 goto write_reg; /* Disable flow control */
5147         }
5148         /**
5149          * If link auto negotiation is enabled, flow control needs to
5150          * be configured according to it
5151          */
5152         switch (an_info & I40E_LINK_PAUSE_RXTX) {
5153         case I40E_LINK_PAUSE_RXTX:
5154                 rxfc = 1;
5155                 txfc = 1;
5156                 hw->fc.current_mode = I40E_FC_FULL;
5157                 break;
5158         case I40E_AQ_LINK_PAUSE_RX:
5159                 rxfc = 1;
5160                 hw->fc.current_mode = I40E_FC_RX_PAUSE;
5161                 break;
5162         case I40E_AQ_LINK_PAUSE_TX:
5163                 txfc = 1;
5164                 hw->fc.current_mode = I40E_FC_TX_PAUSE;
5165                 break;
5166         default:
5167                 hw->fc.current_mode = I40E_FC_NONE;
5168                 break;
5169         }
5170
5171 write_reg:
5172         I40E_WRITE_REG(hw, I40E_PRTDCB_FCCFG,
5173                 txfc << I40E_PRTDCB_FCCFG_TFCE_SHIFT);
5174         reg = I40E_READ_REG(hw, I40E_PRTDCB_MFLCN);
5175         reg &= ~I40E_PRTDCB_MFLCN_RFCE_MASK;
5176         reg |= rxfc << I40E_PRTDCB_MFLCN_RFCE_SHIFT;
5177         I40E_WRITE_REG(hw, I40E_PRTDCB_MFLCN, reg);
5178
5179         return ret;
5180 }
5181
5182 /* PF setup */
5183 static int
5184 i40e_pf_setup(struct i40e_pf *pf)
5185 {
5186         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
5187         struct i40e_filter_control_settings settings;
5188         struct i40e_vsi *vsi;
5189         int ret;
5190
5191         /* Clear all stats counters */
5192         pf->offset_loaded = FALSE;
5193         memset(&pf->stats, 0, sizeof(struct i40e_hw_port_stats));
5194         memset(&pf->stats_offset, 0, sizeof(struct i40e_hw_port_stats));
5195         pf->internal_rx_bytes = 0;
5196         pf->internal_tx_bytes = 0;
5197         pf->internal_rx_bytes_offset = 0;
5198         pf->internal_tx_bytes_offset = 0;
5199
5200         ret = i40e_pf_get_switch_config(pf);
5201         if (ret != I40E_SUCCESS) {
5202                 PMD_DRV_LOG(ERR, "Could not get switch config, err %d", ret);
5203                 return ret;
5204         }
5205         if (pf->flags & I40E_FLAG_FDIR) {
5206                 /* make queue allocated first, let FDIR use queue pair 0*/
5207                 ret = i40e_res_pool_alloc(&pf->qp_pool, I40E_DEFAULT_QP_NUM_FDIR);
5208                 if (ret != I40E_FDIR_QUEUE_ID) {
5209                         PMD_DRV_LOG(ERR, "queue allocation fails for FDIR :"
5210                                     " ret =%d", ret);
5211                         pf->flags &= ~I40E_FLAG_FDIR;
5212                 }
5213         }
5214         /*  main VSI setup */
5215         vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, NULL, 0);
5216         if (!vsi) {
5217                 PMD_DRV_LOG(ERR, "Setup of main vsi failed");
5218                 return I40E_ERR_NOT_READY;
5219         }
5220         pf->main_vsi = vsi;
5221
5222         /* Configure filter control */
5223         memset(&settings, 0, sizeof(settings));
5224         if (hw->func_caps.rss_table_size == ETH_RSS_RETA_SIZE_128)
5225                 settings.hash_lut_size = I40E_HASH_LUT_SIZE_128;
5226         else if (hw->func_caps.rss_table_size == ETH_RSS_RETA_SIZE_512)
5227                 settings.hash_lut_size = I40E_HASH_LUT_SIZE_512;
5228         else {
5229                 PMD_DRV_LOG(ERR, "Hash lookup table size (%u) not supported\n",
5230                                                 hw->func_caps.rss_table_size);
5231                 return I40E_ERR_PARAM;
5232         }
5233         PMD_DRV_LOG(INFO, "Hardware capability of hash lookup table "
5234                         "size: %u\n", hw->func_caps.rss_table_size);
5235         pf->hash_lut_size = hw->func_caps.rss_table_size;
5236
5237         /* Enable ethtype and macvlan filters */
5238         settings.enable_ethtype = TRUE;
5239         settings.enable_macvlan = TRUE;
5240         ret = i40e_set_filter_control(hw, &settings);
5241         if (ret)
5242                 PMD_INIT_LOG(WARNING, "setup_pf_filter_control failed: %d",
5243                                                                 ret);
5244
5245         /* Update flow control according to the auto negotiation */
5246         i40e_update_flow_control(hw);
5247
5248         return I40E_SUCCESS;
5249 }
5250
5251 int
5252 i40e_switch_tx_queue(struct i40e_hw *hw, uint16_t q_idx, bool on)
5253 {
5254         uint32_t reg;
5255         uint16_t j;
5256
5257         /**
5258          * Set or clear TX Queue Disable flags,
5259          * which is required by hardware.
5260          */
5261         i40e_pre_tx_queue_cfg(hw, q_idx, on);
5262         rte_delay_us(I40E_PRE_TX_Q_CFG_WAIT_US);
5263
5264         /* Wait until the request is finished */
5265         for (j = 0; j < I40E_CHK_Q_ENA_COUNT; j++) {
5266                 rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
5267                 reg = I40E_READ_REG(hw, I40E_QTX_ENA(q_idx));
5268                 if (!(((reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 0x1) ^
5269                         ((reg >> I40E_QTX_ENA_QENA_STAT_SHIFT)
5270                                                         & 0x1))) {
5271                         break;
5272                 }
5273         }
5274         if (on) {
5275                 if (reg & I40E_QTX_ENA_QENA_STAT_MASK)
5276                         return I40E_SUCCESS; /* already on, skip next steps */
5277
5278                 I40E_WRITE_REG(hw, I40E_QTX_HEAD(q_idx), 0);
5279                 reg |= I40E_QTX_ENA_QENA_REQ_MASK;
5280         } else {
5281                 if (!(reg & I40E_QTX_ENA_QENA_STAT_MASK))
5282                         return I40E_SUCCESS; /* already off, skip next steps */
5283                 reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
5284         }
5285         /* Write the register */
5286         I40E_WRITE_REG(hw, I40E_QTX_ENA(q_idx), reg);
5287         /* Check the result */
5288         for (j = 0; j < I40E_CHK_Q_ENA_COUNT; j++) {
5289                 rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
5290                 reg = I40E_READ_REG(hw, I40E_QTX_ENA(q_idx));
5291                 if (on) {
5292                         if ((reg & I40E_QTX_ENA_QENA_REQ_MASK) &&
5293                                 (reg & I40E_QTX_ENA_QENA_STAT_MASK))
5294                                 break;
5295                 } else {
5296                         if (!(reg & I40E_QTX_ENA_QENA_REQ_MASK) &&
5297                                 !(reg & I40E_QTX_ENA_QENA_STAT_MASK))
5298                                 break;
5299                 }
5300         }
5301         /* Check if it is timeout */
5302         if (j >= I40E_CHK_Q_ENA_COUNT) {
5303                 PMD_DRV_LOG(ERR, "Failed to %s tx queue[%u]",
5304                             (on ? "enable" : "disable"), q_idx);
5305                 return I40E_ERR_TIMEOUT;
5306         }
5307
5308         return I40E_SUCCESS;
5309 }
5310
5311 /* Swith on or off the tx queues */
5312 static int
5313 i40e_dev_switch_tx_queues(struct i40e_pf *pf, bool on)
5314 {
5315         struct rte_eth_dev_data *dev_data = pf->dev_data;
5316         struct i40e_tx_queue *txq;
5317         struct rte_eth_dev *dev = pf->adapter->eth_dev;
5318         uint16_t i;
5319         int ret;
5320
5321         for (i = 0; i < dev_data->nb_tx_queues; i++) {
5322                 txq = dev_data->tx_queues[i];
5323                 /* Don't operate the queue if not configured or
5324                  * if starting only per queue */
5325                 if (!txq || !txq->q_set || (on && txq->tx_deferred_start))
5326                         continue;
5327                 if (on)
5328                         ret = i40e_dev_tx_queue_start(dev, i);
5329                 else
5330                         ret = i40e_dev_tx_queue_stop(dev, i);
5331                 if ( ret != I40E_SUCCESS)
5332                         return ret;
5333         }
5334
5335         return I40E_SUCCESS;
5336 }
5337
5338 int
5339 i40e_switch_rx_queue(struct i40e_hw *hw, uint16_t q_idx, bool on)
5340 {
5341         uint32_t reg;
5342         uint16_t j;
5343
5344         /* Wait until the request is finished */
5345         for (j = 0; j < I40E_CHK_Q_ENA_COUNT; j++) {
5346                 rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
5347                 reg = I40E_READ_REG(hw, I40E_QRX_ENA(q_idx));
5348                 if (!((reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 0x1) ^
5349                         ((reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 0x1))
5350                         break;
5351         }
5352
5353         if (on) {
5354                 if (reg & I40E_QRX_ENA_QENA_STAT_MASK)
5355                         return I40E_SUCCESS; /* Already on, skip next steps */
5356                 reg |= I40E_QRX_ENA_QENA_REQ_MASK;
5357         } else {
5358                 if (!(reg & I40E_QRX_ENA_QENA_STAT_MASK))
5359                         return I40E_SUCCESS; /* Already off, skip next steps */
5360                 reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
5361         }
5362
5363         /* Write the register */
5364         I40E_WRITE_REG(hw, I40E_QRX_ENA(q_idx), reg);
5365         /* Check the result */
5366         for (j = 0; j < I40E_CHK_Q_ENA_COUNT; j++) {
5367                 rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
5368                 reg = I40E_READ_REG(hw, I40E_QRX_ENA(q_idx));
5369                 if (on) {
5370                         if ((reg & I40E_QRX_ENA_QENA_REQ_MASK) &&
5371                                 (reg & I40E_QRX_ENA_QENA_STAT_MASK))
5372                                 break;
5373                 } else {
5374                         if (!(reg & I40E_QRX_ENA_QENA_REQ_MASK) &&
5375                                 !(reg & I40E_QRX_ENA_QENA_STAT_MASK))
5376                                 break;
5377                 }
5378         }
5379
5380         /* Check if it is timeout */
5381         if (j >= I40E_CHK_Q_ENA_COUNT) {
5382                 PMD_DRV_LOG(ERR, "Failed to %s rx queue[%u]",
5383                             (on ? "enable" : "disable"), q_idx);
5384                 return I40E_ERR_TIMEOUT;
5385         }
5386
5387         return I40E_SUCCESS;
5388 }
5389 /* Switch on or off the rx queues */
5390 static int
5391 i40e_dev_switch_rx_queues(struct i40e_pf *pf, bool on)
5392 {
5393         struct rte_eth_dev_data *dev_data = pf->dev_data;
5394         struct i40e_rx_queue *rxq;
5395         struct rte_eth_dev *dev = pf->adapter->eth_dev;
5396         uint16_t i;
5397         int ret;
5398
5399         for (i = 0; i < dev_data->nb_rx_queues; i++) {
5400                 rxq = dev_data->rx_queues[i];
5401                 /* Don't operate the queue if not configured or
5402                  * if starting only per queue */
5403                 if (!rxq || !rxq->q_set || (on && rxq->rx_deferred_start))
5404                         continue;
5405                 if (on)
5406                         ret = i40e_dev_rx_queue_start(dev, i);
5407                 else
5408                         ret = i40e_dev_rx_queue_stop(dev, i);
5409                 if (ret != I40E_SUCCESS)
5410                         return ret;
5411         }
5412
5413         return I40E_SUCCESS;
5414 }
5415
5416 /* Switch on or off all the rx/tx queues */
5417 int
5418 i40e_dev_switch_queues(struct i40e_pf *pf, bool on)
5419 {
5420         int ret;
5421
5422         if (on) {
5423                 /* enable rx queues before enabling tx queues */
5424                 ret = i40e_dev_switch_rx_queues(pf, on);
5425                 if (ret) {
5426                         PMD_DRV_LOG(ERR, "Failed to switch rx queues");
5427                         return ret;
5428                 }
5429                 ret = i40e_dev_switch_tx_queues(pf, on);
5430         } else {
5431                 /* Stop tx queues before stopping rx queues */
5432                 ret = i40e_dev_switch_tx_queues(pf, on);
5433                 if (ret) {
5434                         PMD_DRV_LOG(ERR, "Failed to switch tx queues");
5435                         return ret;
5436                 }
5437                 ret = i40e_dev_switch_rx_queues(pf, on);
5438         }
5439
5440         return ret;
5441 }
5442
5443 /* Initialize VSI for TX */
5444 static int
5445 i40e_dev_tx_init(struct i40e_pf *pf)
5446 {
5447         struct rte_eth_dev_data *data = pf->dev_data;
5448         uint16_t i;
5449         uint32_t ret = I40E_SUCCESS;
5450         struct i40e_tx_queue *txq;
5451
5452         for (i = 0; i < data->nb_tx_queues; i++) {
5453                 txq = data->tx_queues[i];
5454                 if (!txq || !txq->q_set)
5455                         continue;
5456                 ret = i40e_tx_queue_init(txq);
5457                 if (ret != I40E_SUCCESS)
5458                         break;
5459         }
5460         if (ret == I40E_SUCCESS)
5461                 i40e_set_tx_function(container_of(pf, struct i40e_adapter, pf)
5462                                      ->eth_dev);
5463
5464         return ret;
5465 }
5466
5467 /* Initialize VSI for RX */
5468 static int
5469 i40e_dev_rx_init(struct i40e_pf *pf)
5470 {
5471         struct rte_eth_dev_data *data = pf->dev_data;
5472         int ret = I40E_SUCCESS;
5473         uint16_t i;
5474         struct i40e_rx_queue *rxq;
5475
5476         i40e_pf_config_mq_rx(pf);
5477         for (i = 0; i < data->nb_rx_queues; i++) {
5478                 rxq = data->rx_queues[i];
5479                 if (!rxq || !rxq->q_set)
5480                         continue;
5481
5482                 ret = i40e_rx_queue_init(rxq);
5483                 if (ret != I40E_SUCCESS) {
5484                         PMD_DRV_LOG(ERR, "Failed to do RX queue "
5485                                     "initialization");
5486                         break;
5487                 }
5488         }
5489         if (ret == I40E_SUCCESS)
5490                 i40e_set_rx_function(container_of(pf, struct i40e_adapter, pf)
5491                                      ->eth_dev);
5492
5493         return ret;
5494 }
5495
5496 static int
5497 i40e_dev_rxtx_init(struct i40e_pf *pf)
5498 {
5499         int err;
5500
5501         err = i40e_dev_tx_init(pf);
5502         if (err) {
5503                 PMD_DRV_LOG(ERR, "Failed to do TX initialization");
5504                 return err;
5505         }
5506         err = i40e_dev_rx_init(pf);
5507         if (err) {
5508                 PMD_DRV_LOG(ERR, "Failed to do RX initialization");
5509                 return err;
5510         }
5511
5512         return err;
5513 }
5514
5515 static int
5516 i40e_vmdq_setup(struct rte_eth_dev *dev)
5517 {
5518         struct rte_eth_conf *conf = &dev->data->dev_conf;
5519         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
5520         int i, err, conf_vsis, j, loop;
5521         struct i40e_vsi *vsi;
5522         struct i40e_vmdq_info *vmdq_info;
5523         struct rte_eth_vmdq_rx_conf *vmdq_conf;
5524         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
5525
5526         /*
5527          * Disable interrupt to avoid message from VF. Furthermore, it will
5528          * avoid race condition in VSI creation/destroy.
5529          */
5530         i40e_pf_disable_irq0(hw);
5531
5532         if ((pf->flags & I40E_FLAG_VMDQ) == 0) {
5533                 PMD_INIT_LOG(ERR, "FW doesn't support VMDQ");
5534                 return -ENOTSUP;
5535         }
5536
5537         conf_vsis = conf->rx_adv_conf.vmdq_rx_conf.nb_queue_pools;
5538         if (conf_vsis > pf->max_nb_vmdq_vsi) {
5539                 PMD_INIT_LOG(ERR, "VMDQ config: %u, max support:%u",
5540                         conf->rx_adv_conf.vmdq_rx_conf.nb_queue_pools,
5541                         pf->max_nb_vmdq_vsi);
5542                 return -ENOTSUP;
5543         }
5544
5545         if (pf->vmdq != NULL) {
5546                 PMD_INIT_LOG(INFO, "VMDQ already configured");
5547                 return 0;
5548         }
5549
5550         pf->vmdq = rte_zmalloc("vmdq_info_struct",
5551                                 sizeof(*vmdq_info) * conf_vsis, 0);
5552
5553         if (pf->vmdq == NULL) {
5554                 PMD_INIT_LOG(ERR, "Failed to allocate memory");
5555                 return -ENOMEM;
5556         }
5557
5558         vmdq_conf = &conf->rx_adv_conf.vmdq_rx_conf;
5559
5560         /* Create VMDQ VSI */
5561         for (i = 0; i < conf_vsis; i++) {
5562                 vsi = i40e_vsi_setup(pf, I40E_VSI_VMDQ2, pf->main_vsi,
5563                                 vmdq_conf->enable_loop_back);
5564                 if (vsi == NULL) {
5565                         PMD_INIT_LOG(ERR, "Failed to create VMDQ VSI");
5566                         err = -1;
5567                         goto err_vsi_setup;
5568                 }
5569                 vmdq_info = &pf->vmdq[i];
5570                 vmdq_info->pf = pf;
5571                 vmdq_info->vsi = vsi;
5572         }
5573         pf->nb_cfg_vmdq_vsi = conf_vsis;
5574
5575         /* Configure Vlan */
5576         loop = sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT;
5577         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
5578                 for (j = 0; j < loop && j < pf->nb_cfg_vmdq_vsi; j++) {
5579                         if (vmdq_conf->pool_map[i].pools & (1UL << j)) {
5580                                 PMD_INIT_LOG(INFO, "Add vlan %u to vmdq pool %u",
5581                                         vmdq_conf->pool_map[i].vlan_id, j);
5582
5583                                 err = i40e_vsi_add_vlan(pf->vmdq[j].vsi,
5584                                                 vmdq_conf->pool_map[i].vlan_id);
5585                                 if (err) {
5586                                         PMD_INIT_LOG(ERR, "Failed to add vlan");
5587                                         err = -1;
5588                                         goto err_vsi_setup;
5589                                 }
5590                         }
5591                 }
5592         }
5593
5594         i40e_pf_enable_irq0(hw);
5595
5596         return 0;
5597
5598 err_vsi_setup:
5599         for (i = 0; i < conf_vsis; i++)
5600                 if (pf->vmdq[i].vsi == NULL)
5601                         break;
5602                 else
5603                         i40e_vsi_release(pf->vmdq[i].vsi);
5604
5605         rte_free(pf->vmdq);
5606         pf->vmdq = NULL;
5607         i40e_pf_enable_irq0(hw);
5608         return err;
5609 }
5610
5611 static void
5612 i40e_stat_update_32(struct i40e_hw *hw,
5613                    uint32_t reg,
5614                    bool offset_loaded,
5615                    uint64_t *offset,
5616                    uint64_t *stat)
5617 {
5618         uint64_t new_data;
5619
5620         new_data = (uint64_t)I40E_READ_REG(hw, reg);
5621         if (!offset_loaded)
5622                 *offset = new_data;
5623
5624         if (new_data >= *offset)
5625                 *stat = (uint64_t)(new_data - *offset);
5626         else
5627                 *stat = (uint64_t)((new_data +
5628                         ((uint64_t)1 << I40E_32_BIT_WIDTH)) - *offset);
5629 }
5630
5631 static void
5632 i40e_stat_update_48(struct i40e_hw *hw,
5633                    uint32_t hireg,
5634                    uint32_t loreg,
5635                    bool offset_loaded,
5636                    uint64_t *offset,
5637                    uint64_t *stat)
5638 {
5639         uint64_t new_data;
5640
5641         new_data = (uint64_t)I40E_READ_REG(hw, loreg);
5642         new_data |= ((uint64_t)(I40E_READ_REG(hw, hireg) &
5643                         I40E_16_BIT_MASK)) << I40E_32_BIT_WIDTH;
5644
5645         if (!offset_loaded)
5646                 *offset = new_data;
5647
5648         if (new_data >= *offset)
5649                 *stat = new_data - *offset;
5650         else
5651                 *stat = (uint64_t)((new_data +
5652                         ((uint64_t)1 << I40E_48_BIT_WIDTH)) - *offset);
5653
5654         *stat &= I40E_48_BIT_MASK;
5655 }
5656
5657 /* Disable IRQ0 */
5658 void
5659 i40e_pf_disable_irq0(struct i40e_hw *hw)
5660 {
5661         /* Disable all interrupt types */
5662         I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0,
5663                        I40E_PFINT_DYN_CTL0_ITR_INDX_MASK);
5664         I40E_WRITE_FLUSH(hw);
5665 }
5666
5667 /* Enable IRQ0 */
5668 void
5669 i40e_pf_enable_irq0(struct i40e_hw *hw)
5670 {
5671         I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0,
5672                 I40E_PFINT_DYN_CTL0_INTENA_MASK |
5673                 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
5674                 I40E_PFINT_DYN_CTL0_ITR_INDX_MASK);
5675         I40E_WRITE_FLUSH(hw);
5676 }
5677
5678 static void
5679 i40e_pf_config_irq0(struct i40e_hw *hw, bool no_queue)
5680 {
5681         /* read pending request and disable first */
5682         i40e_pf_disable_irq0(hw);
5683         I40E_WRITE_REG(hw, I40E_PFINT_ICR0_ENA, I40E_PFINT_ICR0_ENA_MASK);
5684         I40E_WRITE_REG(hw, I40E_PFINT_STAT_CTL0,
5685                 I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_MASK);
5686
5687         if (no_queue)
5688                 /* Link no queues with irq0 */
5689                 I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0,
5690                                I40E_PFINT_LNKLST0_FIRSTQ_INDX_MASK);
5691 }
5692
5693 static void
5694 i40e_dev_handle_vfr_event(struct rte_eth_dev *dev)
5695 {
5696         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5697         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
5698         int i;
5699         uint16_t abs_vf_id;
5700         uint32_t index, offset, val;
5701
5702         if (!pf->vfs)
5703                 return;
5704         /**
5705          * Try to find which VF trigger a reset, use absolute VF id to access
5706          * since the reg is global register.
5707          */
5708         for (i = 0; i < pf->vf_num; i++) {
5709                 abs_vf_id = hw->func_caps.vf_base_id + i;
5710                 index = abs_vf_id / I40E_UINT32_BIT_SIZE;
5711                 offset = abs_vf_id % I40E_UINT32_BIT_SIZE;
5712                 val = I40E_READ_REG(hw, I40E_GLGEN_VFLRSTAT(index));
5713                 /* VFR event occured */
5714                 if (val & (0x1 << offset)) {
5715                         int ret;
5716
5717                         /* Clear the event first */
5718                         I40E_WRITE_REG(hw, I40E_GLGEN_VFLRSTAT(index),
5719                                                         (0x1 << offset));
5720                         PMD_DRV_LOG(INFO, "VF %u reset occured", abs_vf_id);
5721                         /**
5722                          * Only notify a VF reset event occured,
5723                          * don't trigger another SW reset
5724                          */
5725                         ret = i40e_pf_host_vf_reset(&pf->vfs[i], 0);
5726                         if (ret != I40E_SUCCESS)
5727                                 PMD_DRV_LOG(ERR, "Failed to do VF reset");
5728                 }
5729         }
5730 }
5731
5732 static void
5733 i40e_notify_all_vfs_link_status(struct rte_eth_dev *dev)
5734 {
5735         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
5736         struct i40e_virtchnl_pf_event event;
5737         int i;
5738
5739         event.event = I40E_VIRTCHNL_EVENT_LINK_CHANGE;
5740         event.event_data.link_event.link_status =
5741                 dev->data->dev_link.link_status;
5742         event.event_data.link_event.link_speed =
5743                 (enum i40e_aq_link_speed)dev->data->dev_link.link_speed;
5744
5745         for (i = 0; i < pf->vf_num; i++)
5746                 i40e_pf_host_send_msg_to_vf(&pf->vfs[i], I40E_VIRTCHNL_OP_EVENT,
5747                                 I40E_SUCCESS, (uint8_t *)&event, sizeof(event));
5748 }
5749
5750 static void
5751 i40e_dev_handle_aq_msg(struct rte_eth_dev *dev)
5752 {
5753         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5754         struct i40e_arq_event_info info;
5755         uint16_t pending, opcode;
5756         int ret;
5757
5758         info.buf_len = I40E_AQ_BUF_SZ;
5759         info.msg_buf = rte_zmalloc("msg_buffer", info.buf_len, 0);
5760         if (!info.msg_buf) {
5761                 PMD_DRV_LOG(ERR, "Failed to allocate mem");
5762                 return;
5763         }
5764
5765         pending = 1;
5766         while (pending) {
5767                 ret = i40e_clean_arq_element(hw, &info, &pending);
5768
5769                 if (ret != I40E_SUCCESS) {
5770                         PMD_DRV_LOG(INFO, "Failed to read msg from AdminQ, "
5771                                     "aq_err: %u", hw->aq.asq_last_status);
5772                         break;
5773                 }
5774                 opcode = rte_le_to_cpu_16(info.desc.opcode);
5775
5776                 switch (opcode) {
5777                 case i40e_aqc_opc_send_msg_to_pf:
5778                         /* Refer to i40e_aq_send_msg_to_pf() for argument layout*/
5779                         i40e_pf_host_handle_vf_msg(dev,
5780                                         rte_le_to_cpu_16(info.desc.retval),
5781                                         rte_le_to_cpu_32(info.desc.cookie_high),
5782                                         rte_le_to_cpu_32(info.desc.cookie_low),
5783                                         info.msg_buf,
5784                                         info.msg_len);
5785                         break;
5786                 case i40e_aqc_opc_get_link_status:
5787                         ret = i40e_dev_link_update(dev, 0);
5788                         if (!ret)
5789                                 _rte_eth_dev_callback_process(dev,
5790                                         RTE_ETH_EVENT_INTR_LSC, NULL);
5791                         break;
5792                 default:
5793                         PMD_DRV_LOG(ERR, "Request %u is not supported yet",
5794                                     opcode);
5795                         break;
5796                 }
5797         }
5798         rte_free(info.msg_buf);
5799 }
5800
5801 /**
5802  * Interrupt handler triggered by NIC  for handling
5803  * specific interrupt.
5804  *
5805  * @param handle
5806  *  Pointer to interrupt handle.
5807  * @param param
5808  *  The address of parameter (struct rte_eth_dev *) regsitered before.
5809  *
5810  * @return
5811  *  void
5812  */
5813 static void
5814 i40e_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
5815                            void *param)
5816 {
5817         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
5818         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5819         uint32_t icr0;
5820
5821         /* Disable interrupt */
5822         i40e_pf_disable_irq0(hw);
5823
5824         /* read out interrupt causes */
5825         icr0 = I40E_READ_REG(hw, I40E_PFINT_ICR0);
5826
5827         /* No interrupt event indicated */
5828         if (!(icr0 & I40E_PFINT_ICR0_INTEVENT_MASK)) {
5829                 PMD_DRV_LOG(INFO, "No interrupt event");
5830                 goto done;
5831         }
5832 #ifdef RTE_LIBRTE_I40E_DEBUG_DRIVER
5833         if (icr0 & I40E_PFINT_ICR0_ECC_ERR_MASK)
5834                 PMD_DRV_LOG(ERR, "ICR0: unrecoverable ECC error");
5835         if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK)
5836                 PMD_DRV_LOG(ERR, "ICR0: malicious programming detected");
5837         if (icr0 & I40E_PFINT_ICR0_GRST_MASK)
5838                 PMD_DRV_LOG(INFO, "ICR0: global reset requested");
5839         if (icr0 & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK)
5840                 PMD_DRV_LOG(INFO, "ICR0: PCI exception activated");
5841         if (icr0 & I40E_PFINT_ICR0_STORM_DETECT_MASK)
5842                 PMD_DRV_LOG(INFO, "ICR0: a change in the storm control state");
5843         if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK)
5844                 PMD_DRV_LOG(ERR, "ICR0: HMC error");
5845         if (icr0 & I40E_PFINT_ICR0_PE_CRITERR_MASK)
5846                 PMD_DRV_LOG(ERR, "ICR0: protocol engine critical error");
5847 #endif /* RTE_LIBRTE_I40E_DEBUG_DRIVER */
5848
5849         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
5850                 PMD_DRV_LOG(INFO, "ICR0: VF reset detected");
5851                 i40e_dev_handle_vfr_event(dev);
5852         }
5853         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
5854                 PMD_DRV_LOG(INFO, "ICR0: adminq event");
5855                 i40e_dev_handle_aq_msg(dev);
5856         }
5857
5858 done:
5859         /* Enable interrupt */
5860         i40e_pf_enable_irq0(hw);
5861         rte_intr_enable(&(dev->pci_dev->intr_handle));
5862 }
5863
5864 static int
5865 i40e_add_macvlan_filters(struct i40e_vsi *vsi,
5866                          struct i40e_macvlan_filter *filter,
5867                          int total)
5868 {
5869         int ele_num, ele_buff_size;
5870         int num, actual_num, i;
5871         uint16_t flags;
5872         int ret = I40E_SUCCESS;
5873         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
5874         struct i40e_aqc_add_macvlan_element_data *req_list;
5875
5876         if (filter == NULL  || total == 0)
5877                 return I40E_ERR_PARAM;
5878         ele_num = hw->aq.asq_buf_size / sizeof(*req_list);
5879         ele_buff_size = hw->aq.asq_buf_size;
5880
5881         req_list = rte_zmalloc("macvlan_add", ele_buff_size, 0);
5882         if (req_list == NULL) {
5883                 PMD_DRV_LOG(ERR, "Fail to allocate memory");
5884                 return I40E_ERR_NO_MEMORY;
5885         }
5886
5887         num = 0;
5888         do {
5889                 actual_num = (num + ele_num > total) ? (total - num) : ele_num;
5890                 memset(req_list, 0, ele_buff_size);
5891
5892                 for (i = 0; i < actual_num; i++) {
5893                         (void)rte_memcpy(req_list[i].mac_addr,
5894                                 &filter[num + i].macaddr, ETH_ADDR_LEN);
5895                         req_list[i].vlan_tag =
5896                                 rte_cpu_to_le_16(filter[num + i].vlan_id);
5897
5898                         switch (filter[num + i].filter_type) {
5899                         case RTE_MAC_PERFECT_MATCH:
5900                                 flags = I40E_AQC_MACVLAN_ADD_PERFECT_MATCH |
5901                                         I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
5902                                 break;
5903                         case RTE_MACVLAN_PERFECT_MATCH:
5904                                 flags = I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
5905                                 break;
5906                         case RTE_MAC_HASH_MATCH:
5907                                 flags = I40E_AQC_MACVLAN_ADD_HASH_MATCH |
5908                                         I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
5909                                 break;
5910                         case RTE_MACVLAN_HASH_MATCH:
5911                                 flags = I40E_AQC_MACVLAN_ADD_HASH_MATCH;
5912                                 break;
5913                         default:
5914                                 PMD_DRV_LOG(ERR, "Invalid MAC match type\n");
5915                                 ret = I40E_ERR_PARAM;
5916                                 goto DONE;
5917                         }
5918
5919                         req_list[i].queue_number = 0;
5920
5921                         req_list[i].flags = rte_cpu_to_le_16(flags);
5922                 }
5923
5924                 ret = i40e_aq_add_macvlan(hw, vsi->seid, req_list,
5925                                                 actual_num, NULL);
5926                 if (ret != I40E_SUCCESS) {
5927                         PMD_DRV_LOG(ERR, "Failed to add macvlan filter");
5928                         goto DONE;
5929                 }
5930                 num += actual_num;
5931         } while (num < total);
5932
5933 DONE:
5934         rte_free(req_list);
5935         return ret;
5936 }
5937
5938 static int
5939 i40e_remove_macvlan_filters(struct i40e_vsi *vsi,
5940                             struct i40e_macvlan_filter *filter,
5941                             int total)
5942 {
5943         int ele_num, ele_buff_size;
5944         int num, actual_num, i;
5945         uint16_t flags;
5946         int ret = I40E_SUCCESS;
5947         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
5948         struct i40e_aqc_remove_macvlan_element_data *req_list;
5949
5950         if (filter == NULL  || total == 0)
5951                 return I40E_ERR_PARAM;
5952
5953         ele_num = hw->aq.asq_buf_size / sizeof(*req_list);
5954         ele_buff_size = hw->aq.asq_buf_size;
5955
5956         req_list = rte_zmalloc("macvlan_remove", ele_buff_size, 0);
5957         if (req_list == NULL) {
5958                 PMD_DRV_LOG(ERR, "Fail to allocate memory");
5959                 return I40E_ERR_NO_MEMORY;
5960         }
5961
5962         num = 0;
5963         do {
5964                 actual_num = (num + ele_num > total) ? (total - num) : ele_num;
5965                 memset(req_list, 0, ele_buff_size);
5966
5967                 for (i = 0; i < actual_num; i++) {
5968                         (void)rte_memcpy(req_list[i].mac_addr,
5969                                 &filter[num + i].macaddr, ETH_ADDR_LEN);
5970                         req_list[i].vlan_tag =
5971                                 rte_cpu_to_le_16(filter[num + i].vlan_id);
5972
5973                         switch (filter[num + i].filter_type) {
5974                         case RTE_MAC_PERFECT_MATCH:
5975                                 flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
5976                                         I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
5977                                 break;
5978                         case RTE_MACVLAN_PERFECT_MATCH:
5979                                 flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
5980                                 break;
5981                         case RTE_MAC_HASH_MATCH:
5982                                 flags = I40E_AQC_MACVLAN_DEL_HASH_MATCH |
5983                                         I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
5984                                 break;
5985                         case RTE_MACVLAN_HASH_MATCH:
5986                                 flags = I40E_AQC_MACVLAN_DEL_HASH_MATCH;
5987                                 break;
5988                         default:
5989                                 PMD_DRV_LOG(ERR, "Invalid MAC filter type\n");
5990                                 ret = I40E_ERR_PARAM;
5991                                 goto DONE;
5992                         }
5993                         req_list[i].flags = rte_cpu_to_le_16(flags);
5994                 }
5995
5996                 ret = i40e_aq_remove_macvlan(hw, vsi->seid, req_list,
5997                                                 actual_num, NULL);
5998                 if (ret != I40E_SUCCESS) {
5999                         PMD_DRV_LOG(ERR, "Failed to remove macvlan filter");
6000                         goto DONE;
6001                 }
6002                 num += actual_num;
6003         } while (num < total);
6004
6005 DONE:
6006         rte_free(req_list);
6007         return ret;
6008 }
6009
6010 /* Find out specific MAC filter */
6011 static struct i40e_mac_filter *
6012 i40e_find_mac_filter(struct i40e_vsi *vsi,
6013                          struct ether_addr *macaddr)
6014 {
6015         struct i40e_mac_filter *f;
6016
6017         TAILQ_FOREACH(f, &vsi->mac_list, next) {
6018                 if (is_same_ether_addr(macaddr, &f->mac_info.mac_addr))
6019                         return f;
6020         }
6021
6022         return NULL;
6023 }
6024
6025 static bool
6026 i40e_find_vlan_filter(struct i40e_vsi *vsi,
6027                          uint16_t vlan_id)
6028 {
6029         uint32_t vid_idx, vid_bit;
6030
6031         if (vlan_id > ETH_VLAN_ID_MAX)
6032                 return 0;
6033
6034         vid_idx = I40E_VFTA_IDX(vlan_id);
6035         vid_bit = I40E_VFTA_BIT(vlan_id);
6036
6037         if (vsi->vfta[vid_idx] & vid_bit)
6038                 return 1;
6039         else
6040                 return 0;
6041 }
6042
6043 static void
6044 i40e_set_vlan_filter(struct i40e_vsi *vsi,
6045                          uint16_t vlan_id, bool on)
6046 {
6047         uint32_t vid_idx, vid_bit;
6048
6049         if (vlan_id > ETH_VLAN_ID_MAX)
6050                 return;
6051
6052         vid_idx = I40E_VFTA_IDX(vlan_id);
6053         vid_bit = I40E_VFTA_BIT(vlan_id);
6054
6055         if (on)
6056                 vsi->vfta[vid_idx] |= vid_bit;
6057         else
6058                 vsi->vfta[vid_idx] &= ~vid_bit;
6059 }
6060
6061 /**
6062  * Find all vlan options for specific mac addr,
6063  * return with actual vlan found.
6064  */
6065 static inline int
6066 i40e_find_all_vlan_for_mac(struct i40e_vsi *vsi,
6067                            struct i40e_macvlan_filter *mv_f,
6068                            int num, struct ether_addr *addr)
6069 {
6070         int i;
6071         uint32_t j, k;
6072
6073         /**
6074          * Not to use i40e_find_vlan_filter to decrease the loop time,
6075          * although the code looks complex.
6076           */
6077         if (num < vsi->vlan_num)
6078                 return I40E_ERR_PARAM;
6079
6080         i = 0;
6081         for (j = 0; j < I40E_VFTA_SIZE; j++) {
6082                 if (vsi->vfta[j]) {
6083                         for (k = 0; k < I40E_UINT32_BIT_SIZE; k++) {
6084                                 if (vsi->vfta[j] & (1 << k)) {
6085                                         if (i > num - 1) {
6086                                                 PMD_DRV_LOG(ERR, "vlan number "
6087                                                             "not match");
6088                                                 return I40E_ERR_PARAM;
6089                                         }
6090                                         (void)rte_memcpy(&mv_f[i].macaddr,
6091                                                         addr, ETH_ADDR_LEN);
6092                                         mv_f[i].vlan_id =
6093                                                 j * I40E_UINT32_BIT_SIZE + k;
6094                                         i++;
6095                                 }
6096                         }
6097                 }
6098         }
6099         return I40E_SUCCESS;
6100 }
6101
6102 static inline int
6103 i40e_find_all_mac_for_vlan(struct i40e_vsi *vsi,
6104                            struct i40e_macvlan_filter *mv_f,
6105                            int num,
6106                            uint16_t vlan)
6107 {
6108         int i = 0;
6109         struct i40e_mac_filter *f;
6110
6111         if (num < vsi->mac_num)
6112                 return I40E_ERR_PARAM;
6113
6114         TAILQ_FOREACH(f, &vsi->mac_list, next) {
6115                 if (i > num - 1) {
6116                         PMD_DRV_LOG(ERR, "buffer number not match");
6117                         return I40E_ERR_PARAM;
6118                 }
6119                 (void)rte_memcpy(&mv_f[i].macaddr, &f->mac_info.mac_addr,
6120                                 ETH_ADDR_LEN);
6121                 mv_f[i].vlan_id = vlan;
6122                 mv_f[i].filter_type = f->mac_info.filter_type;
6123                 i++;
6124         }
6125
6126         return I40E_SUCCESS;
6127 }
6128
6129 static int
6130 i40e_vsi_remove_all_macvlan_filter(struct i40e_vsi *vsi)
6131 {
6132         int i, j, num;
6133         struct i40e_mac_filter *f;
6134         struct i40e_macvlan_filter *mv_f;
6135         int ret = I40E_SUCCESS;
6136
6137         if (vsi == NULL || vsi->mac_num == 0)
6138                 return I40E_ERR_PARAM;
6139
6140         /* Case that no vlan is set */
6141         if (vsi->vlan_num == 0)
6142                 num = vsi->mac_num;
6143         else
6144                 num = vsi->mac_num * vsi->vlan_num;
6145
6146         mv_f = rte_zmalloc("macvlan_data", num * sizeof(*mv_f), 0);
6147         if (mv_f == NULL) {
6148                 PMD_DRV_LOG(ERR, "failed to allocate memory");
6149                 return I40E_ERR_NO_MEMORY;
6150         }
6151
6152         i = 0;
6153         if (vsi->vlan_num == 0) {
6154                 TAILQ_FOREACH(f, &vsi->mac_list, next) {
6155                         (void)rte_memcpy(&mv_f[i].macaddr,
6156                                 &f->mac_info.mac_addr, ETH_ADDR_LEN);
6157                         mv_f[i].filter_type = f->mac_info.filter_type;
6158                         mv_f[i].vlan_id = 0;
6159                         i++;
6160                 }
6161         } else {
6162                 TAILQ_FOREACH(f, &vsi->mac_list, next) {
6163                         ret = i40e_find_all_vlan_for_mac(vsi,&mv_f[i],
6164                                         vsi->vlan_num, &f->mac_info.mac_addr);
6165                         if (ret != I40E_SUCCESS)
6166                                 goto DONE;
6167                         for (j = i; j < i + vsi->vlan_num; j++)
6168                                 mv_f[j].filter_type = f->mac_info.filter_type;
6169                         i += vsi->vlan_num;
6170                 }
6171         }
6172
6173         ret = i40e_remove_macvlan_filters(vsi, mv_f, num);
6174 DONE:
6175         rte_free(mv_f);
6176
6177         return ret;
6178 }
6179
6180 int
6181 i40e_vsi_add_vlan(struct i40e_vsi *vsi, uint16_t vlan)
6182 {
6183         struct i40e_macvlan_filter *mv_f;
6184         int mac_num;
6185         int ret = I40E_SUCCESS;
6186
6187         if (!vsi || vlan > ETHER_MAX_VLAN_ID)
6188                 return I40E_ERR_PARAM;
6189
6190         /* If it's already set, just return */
6191         if (i40e_find_vlan_filter(vsi,vlan))
6192                 return I40E_SUCCESS;
6193
6194         mac_num = vsi->mac_num;
6195
6196         if (mac_num == 0) {
6197                 PMD_DRV_LOG(ERR, "Error! VSI doesn't have a mac addr");
6198                 return I40E_ERR_PARAM;
6199         }
6200
6201         mv_f = rte_zmalloc("macvlan_data", mac_num * sizeof(*mv_f), 0);
6202
6203         if (mv_f == NULL) {
6204                 PMD_DRV_LOG(ERR, "failed to allocate memory");
6205                 return I40E_ERR_NO_MEMORY;
6206         }
6207
6208         ret = i40e_find_all_mac_for_vlan(vsi, mv_f, mac_num, vlan);
6209
6210         if (ret != I40E_SUCCESS)
6211                 goto DONE;
6212
6213         ret = i40e_add_macvlan_filters(vsi, mv_f, mac_num);
6214
6215         if (ret != I40E_SUCCESS)
6216                 goto DONE;
6217
6218         i40e_set_vlan_filter(vsi, vlan, 1);
6219
6220         vsi->vlan_num++;
6221         ret = I40E_SUCCESS;
6222 DONE:
6223         rte_free(mv_f);
6224         return ret;
6225 }
6226
6227 int
6228 i40e_vsi_delete_vlan(struct i40e_vsi *vsi, uint16_t vlan)
6229 {
6230         struct i40e_macvlan_filter *mv_f;
6231         int mac_num;
6232         int ret = I40E_SUCCESS;
6233
6234         /**
6235          * Vlan 0 is the generic filter for untagged packets
6236          * and can't be removed.
6237          */
6238         if (!vsi || vlan == 0 || vlan > ETHER_MAX_VLAN_ID)
6239                 return I40E_ERR_PARAM;
6240
6241         /* If can't find it, just return */
6242         if (!i40e_find_vlan_filter(vsi, vlan))
6243                 return I40E_ERR_PARAM;
6244
6245         mac_num = vsi->mac_num;
6246
6247         if (mac_num == 0) {
6248                 PMD_DRV_LOG(ERR, "Error! VSI doesn't have a mac addr");
6249                 return I40E_ERR_PARAM;
6250         }
6251
6252         mv_f = rte_zmalloc("macvlan_data", mac_num * sizeof(*mv_f), 0);
6253
6254         if (mv_f == NULL) {
6255                 PMD_DRV_LOG(ERR, "failed to allocate memory");
6256                 return I40E_ERR_NO_MEMORY;
6257         }
6258
6259         ret = i40e_find_all_mac_for_vlan(vsi, mv_f, mac_num, vlan);
6260
6261         if (ret != I40E_SUCCESS)
6262                 goto DONE;
6263
6264         ret = i40e_remove_macvlan_filters(vsi, mv_f, mac_num);
6265
6266         if (ret != I40E_SUCCESS)
6267                 goto DONE;
6268
6269         /* This is last vlan to remove, replace all mac filter with vlan 0 */
6270         if (vsi->vlan_num == 1) {
6271                 ret = i40e_find_all_mac_for_vlan(vsi, mv_f, mac_num, 0);
6272                 if (ret != I40E_SUCCESS)
6273                         goto DONE;
6274
6275                 ret = i40e_add_macvlan_filters(vsi, mv_f, mac_num);
6276                 if (ret != I40E_SUCCESS)
6277                         goto DONE;
6278         }
6279
6280         i40e_set_vlan_filter(vsi, vlan, 0);
6281
6282         vsi->vlan_num--;
6283         ret = I40E_SUCCESS;
6284 DONE:
6285         rte_free(mv_f);
6286         return ret;
6287 }
6288
6289 int
6290 i40e_vsi_add_mac(struct i40e_vsi *vsi, struct i40e_mac_filter_info *mac_filter)
6291 {
6292         struct i40e_mac_filter *f;
6293         struct i40e_macvlan_filter *mv_f;
6294         int i, vlan_num = 0;
6295         int ret = I40E_SUCCESS;
6296
6297         /* If it's add and we've config it, return */
6298         f = i40e_find_mac_filter(vsi, &mac_filter->mac_addr);
6299         if (f != NULL)
6300                 return I40E_SUCCESS;
6301         if ((mac_filter->filter_type == RTE_MACVLAN_PERFECT_MATCH) ||
6302                 (mac_filter->filter_type == RTE_MACVLAN_HASH_MATCH)) {
6303
6304                 /**
6305                  * If vlan_num is 0, that's the first time to add mac,
6306                  * set mask for vlan_id 0.
6307                  */
6308                 if (vsi->vlan_num == 0) {
6309                         i40e_set_vlan_filter(vsi, 0, 1);
6310                         vsi->vlan_num = 1;
6311                 }
6312                 vlan_num = vsi->vlan_num;
6313         } else if ((mac_filter->filter_type == RTE_MAC_PERFECT_MATCH) ||
6314                         (mac_filter->filter_type == RTE_MAC_HASH_MATCH))
6315                 vlan_num = 1;
6316
6317         mv_f = rte_zmalloc("macvlan_data", vlan_num * sizeof(*mv_f), 0);
6318         if (mv_f == NULL) {
6319                 PMD_DRV_LOG(ERR, "failed to allocate memory");
6320                 return I40E_ERR_NO_MEMORY;
6321         }
6322
6323         for (i = 0; i < vlan_num; i++) {
6324                 mv_f[i].filter_type = mac_filter->filter_type;
6325                 (void)rte_memcpy(&mv_f[i].macaddr, &mac_filter->mac_addr,
6326                                 ETH_ADDR_LEN);
6327         }
6328
6329         if (mac_filter->filter_type == RTE_MACVLAN_PERFECT_MATCH ||
6330                 mac_filter->filter_type == RTE_MACVLAN_HASH_MATCH) {
6331                 ret = i40e_find_all_vlan_for_mac(vsi, mv_f, vlan_num,
6332                                         &mac_filter->mac_addr);
6333                 if (ret != I40E_SUCCESS)
6334                         goto DONE;
6335         }
6336
6337         ret = i40e_add_macvlan_filters(vsi, mv_f, vlan_num);
6338         if (ret != I40E_SUCCESS)
6339                 goto DONE;
6340
6341         /* Add the mac addr into mac list */
6342         f = rte_zmalloc("macv_filter", sizeof(*f), 0);
6343         if (f == NULL) {
6344                 PMD_DRV_LOG(ERR, "failed to allocate memory");
6345                 ret = I40E_ERR_NO_MEMORY;
6346                 goto DONE;
6347         }
6348         (void)rte_memcpy(&f->mac_info.mac_addr, &mac_filter->mac_addr,
6349                         ETH_ADDR_LEN);
6350         f->mac_info.filter_type = mac_filter->filter_type;
6351         TAILQ_INSERT_TAIL(&vsi->mac_list, f, next);
6352         vsi->mac_num++;
6353
6354         ret = I40E_SUCCESS;
6355 DONE:
6356         rte_free(mv_f);
6357
6358         return ret;
6359 }
6360
6361 int
6362 i40e_vsi_delete_mac(struct i40e_vsi *vsi, struct ether_addr *addr)
6363 {
6364         struct i40e_mac_filter *f;
6365         struct i40e_macvlan_filter *mv_f;
6366         int i, vlan_num;
6367         enum rte_mac_filter_type filter_type;
6368         int ret = I40E_SUCCESS;
6369
6370         /* Can't find it, return an error */
6371         f = i40e_find_mac_filter(vsi, addr);
6372         if (f == NULL)
6373                 return I40E_ERR_PARAM;
6374
6375         vlan_num = vsi->vlan_num;
6376         filter_type = f->mac_info.filter_type;
6377         if (filter_type == RTE_MACVLAN_PERFECT_MATCH ||
6378                 filter_type == RTE_MACVLAN_HASH_MATCH) {
6379                 if (vlan_num == 0) {
6380                         PMD_DRV_LOG(ERR, "VLAN number shouldn't be 0\n");
6381                         return I40E_ERR_PARAM;
6382                 }
6383         } else if (filter_type == RTE_MAC_PERFECT_MATCH ||
6384                         filter_type == RTE_MAC_HASH_MATCH)
6385                 vlan_num = 1;
6386
6387         mv_f = rte_zmalloc("macvlan_data", vlan_num * sizeof(*mv_f), 0);
6388         if (mv_f == NULL) {
6389                 PMD_DRV_LOG(ERR, "failed to allocate memory");
6390                 return I40E_ERR_NO_MEMORY;
6391         }
6392
6393         for (i = 0; i < vlan_num; i++) {
6394                 mv_f[i].filter_type = filter_type;
6395                 (void)rte_memcpy(&mv_f[i].macaddr, &f->mac_info.mac_addr,
6396                                 ETH_ADDR_LEN);
6397         }
6398         if (filter_type == RTE_MACVLAN_PERFECT_MATCH ||
6399                         filter_type == RTE_MACVLAN_HASH_MATCH) {
6400                 ret = i40e_find_all_vlan_for_mac(vsi, mv_f, vlan_num, addr);
6401                 if (ret != I40E_SUCCESS)
6402                         goto DONE;
6403         }
6404
6405         ret = i40e_remove_macvlan_filters(vsi, mv_f, vlan_num);
6406         if (ret != I40E_SUCCESS)
6407                 goto DONE;
6408
6409         /* Remove the mac addr into mac list */
6410         TAILQ_REMOVE(&vsi->mac_list, f, next);
6411         rte_free(f);
6412         vsi->mac_num--;
6413
6414         ret = I40E_SUCCESS;
6415 DONE:
6416         rte_free(mv_f);
6417         return ret;
6418 }
6419
6420 /* Configure hash enable flags for RSS */
6421 uint64_t
6422 i40e_config_hena(uint64_t flags, enum i40e_mac_type type)
6423 {
6424         uint64_t hena = 0;
6425
6426         if (!flags)
6427                 return hena;
6428
6429         if (flags & ETH_RSS_FRAG_IPV4)
6430                 hena |= 1ULL << I40E_FILTER_PCTYPE_FRAG_IPV4;
6431         if (flags & ETH_RSS_NONFRAG_IPV4_TCP) {
6432                 if (type == I40E_MAC_X722) {
6433                         hena |= (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP) |
6434                          (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK);
6435                 } else
6436                         hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP;
6437         }
6438         if (flags & ETH_RSS_NONFRAG_IPV4_UDP) {
6439                 if (type == I40E_MAC_X722) {
6440                         hena |= (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_UDP) |
6441                          (1ULL << I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) |
6442                          (1ULL << I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP);
6443                 } else
6444                         hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_UDP;
6445         }
6446         if (flags & ETH_RSS_NONFRAG_IPV4_SCTP)
6447                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP;
6448         if (flags & ETH_RSS_NONFRAG_IPV4_OTHER)
6449                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER;
6450         if (flags & ETH_RSS_FRAG_IPV6)
6451                 hena |= 1ULL << I40E_FILTER_PCTYPE_FRAG_IPV6;
6452         if (flags & ETH_RSS_NONFRAG_IPV6_TCP) {
6453                 if (type == I40E_MAC_X722) {
6454                         hena |= (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP) |
6455                          (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK);
6456                 } else
6457                         hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP;
6458         }
6459         if (flags & ETH_RSS_NONFRAG_IPV6_UDP) {
6460                 if (type == I40E_MAC_X722) {
6461                         hena |= (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_UDP) |
6462                          (1ULL << I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) |
6463                          (1ULL << I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP);
6464                 } else
6465                         hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_UDP;
6466         }
6467         if (flags & ETH_RSS_NONFRAG_IPV6_SCTP)
6468                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP;
6469         if (flags & ETH_RSS_NONFRAG_IPV6_OTHER)
6470                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER;
6471         if (flags & ETH_RSS_L2_PAYLOAD)
6472                 hena |= 1ULL << I40E_FILTER_PCTYPE_L2_PAYLOAD;
6473
6474         return hena;
6475 }
6476
6477 /* Parse the hash enable flags */
6478 uint64_t
6479 i40e_parse_hena(uint64_t flags)
6480 {
6481         uint64_t rss_hf = 0;
6482
6483         if (!flags)
6484                 return rss_hf;
6485         if (flags & (1ULL << I40E_FILTER_PCTYPE_FRAG_IPV4))
6486                 rss_hf |= ETH_RSS_FRAG_IPV4;
6487         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP))
6488                 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP;
6489 #ifdef X722_SUPPORT
6490         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK))
6491                 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP;
6492 #endif
6493         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_UDP))
6494                 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP;
6495 #ifdef X722_SUPPORT
6496         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP))
6497                 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP;
6498         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP))
6499                 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP;
6500 #endif
6501         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP))
6502                 rss_hf |= ETH_RSS_NONFRAG_IPV4_SCTP;
6503         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER))
6504                 rss_hf |= ETH_RSS_NONFRAG_IPV4_OTHER;
6505         if (flags & (1ULL << I40E_FILTER_PCTYPE_FRAG_IPV6))
6506                 rss_hf |= ETH_RSS_FRAG_IPV6;
6507         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP))
6508                 rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP;
6509 #ifdef X722_SUPPORT
6510         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK))
6511                 rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP;
6512 #endif
6513         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_UDP))
6514                 rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP;
6515 #ifdef X722_SUPPORT
6516         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP))
6517                 rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP;
6518         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP))
6519                 rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP;
6520 #endif
6521         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP))
6522                 rss_hf |= ETH_RSS_NONFRAG_IPV6_SCTP;
6523         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER))
6524                 rss_hf |= ETH_RSS_NONFRAG_IPV6_OTHER;
6525         if (flags & (1ULL << I40E_FILTER_PCTYPE_L2_PAYLOAD))
6526                 rss_hf |= ETH_RSS_L2_PAYLOAD;
6527
6528         return rss_hf;
6529 }
6530
6531 /* Disable RSS */
6532 static void
6533 i40e_pf_disable_rss(struct i40e_pf *pf)
6534 {
6535         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
6536         uint64_t hena;
6537
6538         hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0));
6539         hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1))) << 32;
6540         if (hw->mac.type == I40E_MAC_X722)
6541                 hena &= ~I40E_RSS_HENA_ALL_X722;
6542         else
6543                 hena &= ~I40E_RSS_HENA_ALL;
6544         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (uint32_t)hena);
6545         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (uint32_t)(hena >> 32));
6546         I40E_WRITE_FLUSH(hw);
6547 }
6548
6549 static int
6550 i40e_set_rss_key(struct i40e_vsi *vsi, uint8_t *key, uint8_t key_len)
6551 {
6552         struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
6553         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
6554         int ret = 0;
6555
6556         if (!key || key_len == 0) {
6557                 PMD_DRV_LOG(DEBUG, "No key to be configured");
6558                 return 0;
6559         } else if (key_len != (I40E_PFQF_HKEY_MAX_INDEX + 1) *
6560                 sizeof(uint32_t)) {
6561                 PMD_DRV_LOG(ERR, "Invalid key length %u", key_len);
6562                 return -EINVAL;
6563         }
6564
6565         if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
6566                 struct i40e_aqc_get_set_rss_key_data *key_dw =
6567                         (struct i40e_aqc_get_set_rss_key_data *)key;
6568
6569                 ret = i40e_aq_set_rss_key(hw, vsi->vsi_id, key_dw);
6570                 if (ret)
6571                         PMD_INIT_LOG(ERR, "Failed to configure RSS key "
6572                                      "via AQ");
6573         } else {
6574                 uint32_t *hash_key = (uint32_t *)key;
6575                 uint16_t i;
6576
6577                 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
6578                         i40e_write_rx_ctl(hw, I40E_PFQF_HKEY(i), hash_key[i]);
6579                 I40E_WRITE_FLUSH(hw);
6580         }
6581
6582         return ret;
6583 }
6584
6585 static int
6586 i40e_get_rss_key(struct i40e_vsi *vsi, uint8_t *key, uint8_t *key_len)
6587 {
6588         struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
6589         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
6590         int ret;
6591
6592         if (!key || !key_len)
6593                 return -EINVAL;
6594
6595         if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
6596                 ret = i40e_aq_get_rss_key(hw, vsi->vsi_id,
6597                         (struct i40e_aqc_get_set_rss_key_data *)key);
6598                 if (ret) {
6599                         PMD_INIT_LOG(ERR, "Failed to get RSS key via AQ");
6600                         return ret;
6601                 }
6602         } else {
6603                 uint32_t *key_dw = (uint32_t *)key;
6604                 uint16_t i;
6605
6606                 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
6607                         key_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i));
6608         }
6609         *key_len = (I40E_PFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t);
6610
6611         return 0;
6612 }
6613
6614 static int
6615 i40e_hw_rss_hash_set(struct i40e_pf *pf, struct rte_eth_rss_conf *rss_conf)
6616 {
6617         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
6618         uint64_t rss_hf;
6619         uint64_t hena;
6620         int ret;
6621
6622         ret = i40e_set_rss_key(pf->main_vsi, rss_conf->rss_key,
6623                                rss_conf->rss_key_len);
6624         if (ret)
6625                 return ret;
6626
6627         rss_hf = rss_conf->rss_hf;
6628         hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0));
6629         hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1))) << 32;
6630         if (hw->mac.type == I40E_MAC_X722)
6631                 hena &= ~I40E_RSS_HENA_ALL_X722;
6632         else
6633                 hena &= ~I40E_RSS_HENA_ALL;
6634         hena |= i40e_config_hena(rss_hf, hw->mac.type);
6635         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (uint32_t)hena);
6636         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (uint32_t)(hena >> 32));
6637         I40E_WRITE_FLUSH(hw);
6638
6639         return 0;
6640 }
6641
6642 static int
6643 i40e_dev_rss_hash_update(struct rte_eth_dev *dev,
6644                          struct rte_eth_rss_conf *rss_conf)
6645 {
6646         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
6647         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
6648         uint64_t rss_hf = rss_conf->rss_hf & I40E_RSS_OFFLOAD_ALL;
6649         uint64_t hena;
6650
6651         hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0));
6652         hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1))) << 32;
6653         if (!(hena & ((hw->mac.type == I40E_MAC_X722)
6654                  ? I40E_RSS_HENA_ALL_X722
6655                  : I40E_RSS_HENA_ALL))) { /* RSS disabled */
6656                 if (rss_hf != 0) /* Enable RSS */
6657                         return -EINVAL;
6658                 return 0; /* Nothing to do */
6659         }
6660         /* RSS enabled */
6661         if (rss_hf == 0) /* Disable RSS */
6662                 return -EINVAL;
6663
6664         return i40e_hw_rss_hash_set(pf, rss_conf);
6665 }
6666
6667 static int
6668 i40e_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
6669                            struct rte_eth_rss_conf *rss_conf)
6670 {
6671         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
6672         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
6673         uint64_t hena;
6674
6675         i40e_get_rss_key(pf->main_vsi, rss_conf->rss_key,
6676                          &rss_conf->rss_key_len);
6677
6678         hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0));
6679         hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1))) << 32;
6680         rss_conf->rss_hf = i40e_parse_hena(hena);
6681
6682         return 0;
6683 }
6684
6685 static int
6686 i40e_dev_get_filter_type(uint16_t filter_type, uint16_t *flag)
6687 {
6688         switch (filter_type) {
6689         case RTE_TUNNEL_FILTER_IMAC_IVLAN:
6690                 *flag = I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN;
6691                 break;
6692         case RTE_TUNNEL_FILTER_IMAC_IVLAN_TENID:
6693                 *flag = I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID;
6694                 break;
6695         case RTE_TUNNEL_FILTER_IMAC_TENID:
6696                 *flag = I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID;
6697                 break;
6698         case RTE_TUNNEL_FILTER_OMAC_TENID_IMAC:
6699                 *flag = I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC;
6700                 break;
6701         case ETH_TUNNEL_FILTER_IMAC:
6702                 *flag = I40E_AQC_ADD_CLOUD_FILTER_IMAC;
6703                 break;
6704         case ETH_TUNNEL_FILTER_OIP:
6705                 *flag = I40E_AQC_ADD_CLOUD_FILTER_OIP;
6706                 break;
6707         case ETH_TUNNEL_FILTER_IIP:
6708                 *flag = I40E_AQC_ADD_CLOUD_FILTER_IIP;
6709                 break;
6710         default:
6711                 PMD_DRV_LOG(ERR, "invalid tunnel filter type");
6712                 return -EINVAL;
6713         }
6714
6715         return 0;
6716 }
6717
6718 static int
6719 i40e_dev_tunnel_filter_set(struct i40e_pf *pf,
6720                         struct rte_eth_tunnel_filter_conf *tunnel_filter,
6721                         uint8_t add)
6722 {
6723         uint16_t ip_type;
6724         uint32_t ipv4_addr, ipv4_addr_le;
6725         uint8_t i, tun_type = 0;
6726         /* internal varialbe to convert ipv6 byte order */
6727         uint32_t convert_ipv6[4];
6728         int val, ret = 0;
6729         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
6730         struct i40e_vsi *vsi = pf->main_vsi;
6731         struct i40e_aqc_add_remove_cloud_filters_element_data  *cld_filter;
6732         struct i40e_aqc_add_remove_cloud_filters_element_data  *pfilter;
6733
6734         cld_filter = rte_zmalloc("tunnel_filter",
6735                 sizeof(struct i40e_aqc_add_remove_cloud_filters_element_data),
6736                 0);
6737
6738         if (NULL == cld_filter) {
6739                 PMD_DRV_LOG(ERR, "Failed to alloc memory.");
6740                 return -EINVAL;
6741         }
6742         pfilter = cld_filter;
6743
6744         ether_addr_copy(&tunnel_filter->outer_mac, (struct ether_addr*)&pfilter->outer_mac);
6745         ether_addr_copy(&tunnel_filter->inner_mac, (struct ether_addr*)&pfilter->inner_mac);
6746
6747         pfilter->inner_vlan = rte_cpu_to_le_16(tunnel_filter->inner_vlan);
6748         if (tunnel_filter->ip_type == RTE_TUNNEL_IPTYPE_IPV4) {
6749                 ip_type = I40E_AQC_ADD_CLOUD_FLAGS_IPV4;
6750                 ipv4_addr = rte_be_to_cpu_32(tunnel_filter->ip_addr.ipv4_addr);
6751                 ipv4_addr_le = rte_cpu_to_le_32(ipv4_addr);
6752                 rte_memcpy(&pfilter->ipaddr.v4.data,
6753                                 &ipv4_addr_le,
6754                                 sizeof(pfilter->ipaddr.v4.data));
6755         } else {
6756                 ip_type = I40E_AQC_ADD_CLOUD_FLAGS_IPV6;
6757                 for (i = 0; i < 4; i++) {
6758                         convert_ipv6[i] =
6759                         rte_cpu_to_le_32(rte_be_to_cpu_32(tunnel_filter->ip_addr.ipv6_addr[i]));
6760                 }
6761                 rte_memcpy(&pfilter->ipaddr.v6.data, &convert_ipv6,
6762                                 sizeof(pfilter->ipaddr.v6.data));
6763         }
6764
6765         /* check tunneled type */
6766         switch (tunnel_filter->tunnel_type) {
6767         case RTE_TUNNEL_TYPE_VXLAN:
6768                 tun_type = I40E_AQC_ADD_CLOUD_TNL_TYPE_VXLAN;
6769                 break;
6770         case RTE_TUNNEL_TYPE_NVGRE:
6771                 tun_type = I40E_AQC_ADD_CLOUD_TNL_TYPE_NVGRE_OMAC;
6772                 break;
6773         case RTE_TUNNEL_TYPE_IP_IN_GRE:
6774                 tun_type = I40E_AQC_ADD_CLOUD_TNL_TYPE_IP;
6775                 break;
6776         default:
6777                 /* Other tunnel types is not supported. */
6778                 PMD_DRV_LOG(ERR, "tunnel type is not supported.");
6779                 rte_free(cld_filter);
6780                 return -EINVAL;
6781         }
6782
6783         val = i40e_dev_get_filter_type(tunnel_filter->filter_type,
6784                                                 &pfilter->flags);
6785         if (val < 0) {
6786                 rte_free(cld_filter);
6787                 return -EINVAL;
6788         }
6789
6790         pfilter->flags |= rte_cpu_to_le_16(
6791                 I40E_AQC_ADD_CLOUD_FLAGS_TO_QUEUE |
6792                 ip_type | (tun_type << I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT));
6793         pfilter->tenant_id = rte_cpu_to_le_32(tunnel_filter->tenant_id);
6794         pfilter->queue_number = rte_cpu_to_le_16(tunnel_filter->queue_id);
6795
6796         if (add)
6797                 ret = i40e_aq_add_cloud_filters(hw, vsi->seid, cld_filter, 1);
6798         else
6799                 ret = i40e_aq_remove_cloud_filters(hw, vsi->seid,
6800                                                 cld_filter, 1);
6801
6802         rte_free(cld_filter);
6803         return ret;
6804 }
6805
6806 static int
6807 i40e_get_vxlan_port_idx(struct i40e_pf *pf, uint16_t port)
6808 {
6809         uint8_t i;
6810
6811         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
6812                 if (pf->vxlan_ports[i] == port)
6813                         return i;
6814         }
6815
6816         return -1;
6817 }
6818
6819 static int
6820 i40e_add_vxlan_port(struct i40e_pf *pf, uint16_t port)
6821 {
6822         int  idx, ret;
6823         uint8_t filter_idx;
6824         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
6825
6826         idx = i40e_get_vxlan_port_idx(pf, port);
6827
6828         /* Check if port already exists */
6829         if (idx >= 0) {
6830                 PMD_DRV_LOG(ERR, "Port %d already offloaded", port);
6831                 return -EINVAL;
6832         }
6833
6834         /* Now check if there is space to add the new port */
6835         idx = i40e_get_vxlan_port_idx(pf, 0);
6836         if (idx < 0) {
6837                 PMD_DRV_LOG(ERR, "Maximum number of UDP ports reached,"
6838                         "not adding port %d", port);
6839                 return -ENOSPC;
6840         }
6841
6842         ret =  i40e_aq_add_udp_tunnel(hw, port, I40E_AQC_TUNNEL_TYPE_VXLAN,
6843                                         &filter_idx, NULL);
6844         if (ret < 0) {
6845                 PMD_DRV_LOG(ERR, "Failed to add VXLAN UDP port %d", port);
6846                 return -1;
6847         }
6848
6849         PMD_DRV_LOG(INFO, "Added port %d with AQ command with index %d",
6850                          port,  filter_idx);
6851
6852         /* New port: add it and mark its index in the bitmap */
6853         pf->vxlan_ports[idx] = port;
6854         pf->vxlan_bitmap |= (1 << idx);
6855
6856         if (!(pf->flags & I40E_FLAG_VXLAN))
6857                 pf->flags |= I40E_FLAG_VXLAN;
6858
6859         return 0;
6860 }
6861
6862 static int
6863 i40e_del_vxlan_port(struct i40e_pf *pf, uint16_t port)
6864 {
6865         int idx;
6866         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
6867
6868         if (!(pf->flags & I40E_FLAG_VXLAN)) {
6869                 PMD_DRV_LOG(ERR, "VXLAN UDP port was not configured.");
6870                 return -EINVAL;
6871         }
6872
6873         idx = i40e_get_vxlan_port_idx(pf, port);
6874
6875         if (idx < 0) {
6876                 PMD_DRV_LOG(ERR, "Port %d doesn't exist", port);
6877                 return -EINVAL;
6878         }
6879
6880         if (i40e_aq_del_udp_tunnel(hw, idx, NULL) < 0) {
6881                 PMD_DRV_LOG(ERR, "Failed to delete VXLAN UDP port %d", port);
6882                 return -1;
6883         }
6884
6885         PMD_DRV_LOG(INFO, "Deleted port %d with AQ command with index %d",
6886                         port, idx);
6887
6888         pf->vxlan_ports[idx] = 0;
6889         pf->vxlan_bitmap &= ~(1 << idx);
6890
6891         if (!pf->vxlan_bitmap)
6892                 pf->flags &= ~I40E_FLAG_VXLAN;
6893
6894         return 0;
6895 }
6896
6897 /* Add UDP tunneling port */
6898 static int
6899 i40e_dev_udp_tunnel_port_add(struct rte_eth_dev *dev,
6900                              struct rte_eth_udp_tunnel *udp_tunnel)
6901 {
6902         int ret = 0;
6903         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
6904
6905         if (udp_tunnel == NULL)
6906                 return -EINVAL;
6907
6908         switch (udp_tunnel->prot_type) {
6909         case RTE_TUNNEL_TYPE_VXLAN:
6910                 ret = i40e_add_vxlan_port(pf, udp_tunnel->udp_port);
6911                 break;
6912
6913         case RTE_TUNNEL_TYPE_GENEVE:
6914         case RTE_TUNNEL_TYPE_TEREDO:
6915                 PMD_DRV_LOG(ERR, "Tunnel type is not supported now.");
6916                 ret = -1;
6917                 break;
6918
6919         default:
6920                 PMD_DRV_LOG(ERR, "Invalid tunnel type");
6921                 ret = -1;
6922                 break;
6923         }
6924
6925         return ret;
6926 }
6927
6928 /* Remove UDP tunneling port */
6929 static int
6930 i40e_dev_udp_tunnel_port_del(struct rte_eth_dev *dev,
6931                              struct rte_eth_udp_tunnel *udp_tunnel)
6932 {
6933         int ret = 0;
6934         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
6935
6936         if (udp_tunnel == NULL)
6937                 return -EINVAL;
6938
6939         switch (udp_tunnel->prot_type) {
6940         case RTE_TUNNEL_TYPE_VXLAN:
6941                 ret = i40e_del_vxlan_port(pf, udp_tunnel->udp_port);
6942                 break;
6943         case RTE_TUNNEL_TYPE_GENEVE:
6944         case RTE_TUNNEL_TYPE_TEREDO:
6945                 PMD_DRV_LOG(ERR, "Tunnel type is not supported now.");
6946                 ret = -1;
6947                 break;
6948         default:
6949                 PMD_DRV_LOG(ERR, "Invalid tunnel type");
6950                 ret = -1;
6951                 break;
6952         }
6953
6954         return ret;
6955 }
6956
6957 /* Calculate the maximum number of contiguous PF queues that are configured */
6958 static int
6959 i40e_pf_calc_configured_queues_num(struct i40e_pf *pf)
6960 {
6961         struct rte_eth_dev_data *data = pf->dev_data;
6962         int i, num;
6963         struct i40e_rx_queue *rxq;
6964
6965         num = 0;
6966         for (i = 0; i < pf->lan_nb_qps; i++) {
6967                 rxq = data->rx_queues[i];
6968                 if (rxq && rxq->q_set)
6969                         num++;
6970                 else
6971                         break;
6972         }
6973
6974         return num;
6975 }
6976
6977 /* Configure RSS */
6978 static int
6979 i40e_pf_config_rss(struct i40e_pf *pf)
6980 {
6981         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
6982         struct rte_eth_rss_conf rss_conf;
6983         uint32_t i, lut = 0;
6984         uint16_t j, num;
6985
6986         /*
6987          * If both VMDQ and RSS enabled, not all of PF queues are configured.
6988          * It's necessary to calulate the actual PF queues that are configured.
6989          */
6990         if (pf->dev_data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
6991                 num = i40e_pf_calc_configured_queues_num(pf);
6992         else
6993                 num = pf->dev_data->nb_rx_queues;
6994
6995         num = RTE_MIN(num, I40E_MAX_Q_PER_TC);
6996         PMD_INIT_LOG(INFO, "Max of contiguous %u PF queues are configured",
6997                         num);
6998
6999         if (num == 0) {
7000                 PMD_INIT_LOG(ERR, "No PF queues are configured to enable RSS");
7001                 return -ENOTSUP;
7002         }
7003
7004         for (i = 0, j = 0; i < hw->func_caps.rss_table_size; i++, j++) {
7005                 if (j == num)
7006                         j = 0;
7007                 lut = (lut << 8) | (j & ((0x1 <<
7008                         hw->func_caps.rss_table_entry_width) - 1));
7009                 if ((i & 3) == 3)
7010                         I40E_WRITE_REG(hw, I40E_PFQF_HLUT(i >> 2), lut);
7011         }
7012
7013         rss_conf = pf->dev_data->dev_conf.rx_adv_conf.rss_conf;
7014         if ((rss_conf.rss_hf & I40E_RSS_OFFLOAD_ALL) == 0) {
7015                 i40e_pf_disable_rss(pf);
7016                 return 0;
7017         }
7018         if (rss_conf.rss_key == NULL || rss_conf.rss_key_len <
7019                 (I40E_PFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t)) {
7020                 /* Random default keys */
7021                 static uint32_t rss_key_default[] = {0x6b793944,
7022                         0x23504cb5, 0x5bea75b6, 0x309f4f12, 0x3dc0a2b8,
7023                         0x024ddcdf, 0x339b8ca0, 0x4c4af64a, 0x34fac605,
7024                         0x55d85839, 0x3a58997d, 0x2ec938e1, 0x66031581};
7025
7026                 rss_conf.rss_key = (uint8_t *)rss_key_default;
7027                 rss_conf.rss_key_len = (I40E_PFQF_HKEY_MAX_INDEX + 1) *
7028                                                         sizeof(uint32_t);
7029         }
7030
7031         return i40e_hw_rss_hash_set(pf, &rss_conf);
7032 }
7033
7034 static int
7035 i40e_tunnel_filter_param_check(struct i40e_pf *pf,
7036                                struct rte_eth_tunnel_filter_conf *filter)
7037 {
7038         if (pf == NULL || filter == NULL) {
7039                 PMD_DRV_LOG(ERR, "Invalid parameter");
7040                 return -EINVAL;
7041         }
7042
7043         if (filter->queue_id >= pf->dev_data->nb_rx_queues) {
7044                 PMD_DRV_LOG(ERR, "Invalid queue ID");
7045                 return -EINVAL;
7046         }
7047
7048         if (filter->inner_vlan > ETHER_MAX_VLAN_ID) {
7049                 PMD_DRV_LOG(ERR, "Invalid inner VLAN ID");
7050                 return -EINVAL;
7051         }
7052
7053         if ((filter->filter_type & ETH_TUNNEL_FILTER_OMAC) &&
7054                 (is_zero_ether_addr(&filter->outer_mac))) {
7055                 PMD_DRV_LOG(ERR, "Cannot add NULL outer MAC address");
7056                 return -EINVAL;
7057         }
7058
7059         if ((filter->filter_type & ETH_TUNNEL_FILTER_IMAC) &&
7060                 (is_zero_ether_addr(&filter->inner_mac))) {
7061                 PMD_DRV_LOG(ERR, "Cannot add NULL inner MAC address");
7062                 return -EINVAL;
7063         }
7064
7065         return 0;
7066 }
7067
7068 #define I40E_GL_PRS_FVBM_MSK_ENA 0x80000000
7069 #define I40E_GL_PRS_FVBM(_i)     (0x00269760 + ((_i) * 4))
7070 static int
7071 i40e_dev_set_gre_key_len(struct i40e_hw *hw, uint8_t len)
7072 {
7073         struct i40e_pf *pf = &((struct i40e_adapter *)hw->back)->pf;
7074         uint32_t val, reg;
7075         int ret = -EINVAL;
7076
7077         if (pf->support_multi_driver) {
7078                 PMD_DRV_LOG(ERR, "GRE key length configuration is unsupported");
7079                 return -ENOTSUP;
7080         }
7081
7082         val = I40E_READ_REG(hw, I40E_GL_PRS_FVBM(2));
7083         PMD_DRV_LOG(DEBUG, "Read original GL_PRS_FVBM with 0x%08x\n", val);
7084
7085         if (len == 3) {
7086                 reg = val | I40E_GL_PRS_FVBM_MSK_ENA;
7087         } else if (len == 4) {
7088                 reg = val & ~I40E_GL_PRS_FVBM_MSK_ENA;
7089         } else {
7090                 PMD_DRV_LOG(ERR, "Unsupported GRE key length of %u", len);
7091                 return ret;
7092         }
7093
7094         if (reg != val) {
7095                 ret = i40e_aq_debug_write_register(hw, I40E_GL_PRS_FVBM(2),
7096                                                    reg, NULL);
7097                 if (ret != 0)
7098                         return ret;
7099                 PMD_DRV_LOG(DEBUG, "Global register 0x%08x is changed "
7100                             "with value 0x%08x",
7101                             I40E_GL_PRS_FVBM(2), reg);
7102                 i40e_global_cfg_warning(I40E_WARNING_GRE_KEY_LEN);
7103         } else {
7104                 ret = 0;
7105         }
7106         PMD_DRV_LOG(DEBUG, "Read modified GL_PRS_FVBM with 0x%08x\n",
7107                     I40E_READ_REG(hw, I40E_GL_PRS_FVBM(2)));
7108
7109         return ret;
7110 }
7111
7112 static int
7113 i40e_dev_global_config_set(struct i40e_hw *hw, struct rte_eth_global_cfg *cfg)
7114 {
7115         int ret = -EINVAL;
7116
7117         if (!hw || !cfg)
7118                 return -EINVAL;
7119
7120         switch (cfg->cfg_type) {
7121         case RTE_ETH_GLOBAL_CFG_TYPE_GRE_KEY_LEN:
7122                 ret = i40e_dev_set_gre_key_len(hw, cfg->cfg.gre_key_len);
7123                 break;
7124         default:
7125                 PMD_DRV_LOG(ERR, "Unknown config type %u", cfg->cfg_type);
7126                 break;
7127         }
7128
7129         return ret;
7130 }
7131
7132 static int
7133 i40e_filter_ctrl_global_config(struct rte_eth_dev *dev,
7134                                enum rte_filter_op filter_op,
7135                                void *arg)
7136 {
7137         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
7138         int ret = I40E_ERR_PARAM;
7139
7140         switch (filter_op) {
7141         case RTE_ETH_FILTER_SET:
7142                 ret = i40e_dev_global_config_set(hw,
7143                         (struct rte_eth_global_cfg *)arg);
7144                 break;
7145         default:
7146                 PMD_DRV_LOG(ERR, "unknown operation %u", filter_op);
7147                 break;
7148         }
7149
7150         return ret;
7151 }
7152
7153 static int
7154 i40e_tunnel_filter_handle(struct rte_eth_dev *dev,
7155                           enum rte_filter_op filter_op,
7156                           void *arg)
7157 {
7158         struct rte_eth_tunnel_filter_conf *filter;
7159         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
7160         int ret = I40E_SUCCESS;
7161
7162         filter = (struct rte_eth_tunnel_filter_conf *)(arg);
7163
7164         if (i40e_tunnel_filter_param_check(pf, filter) < 0)
7165                 return I40E_ERR_PARAM;
7166
7167         switch (filter_op) {
7168         case RTE_ETH_FILTER_NOP:
7169                 if (!(pf->flags & I40E_FLAG_VXLAN))
7170                         ret = I40E_NOT_SUPPORTED;
7171                 break;
7172         case RTE_ETH_FILTER_ADD:
7173                 ret = i40e_dev_tunnel_filter_set(pf, filter, 1);
7174                 break;
7175         case RTE_ETH_FILTER_DELETE:
7176                 ret = i40e_dev_tunnel_filter_set(pf, filter, 0);
7177                 break;
7178         default:
7179                 PMD_DRV_LOG(ERR, "unknown operation %u", filter_op);
7180                 ret = I40E_ERR_PARAM;
7181                 break;
7182         }
7183
7184         return ret;
7185 }
7186
7187 static int
7188 i40e_pf_config_mq_rx(struct i40e_pf *pf)
7189 {
7190         int ret = 0;
7191         enum rte_eth_rx_mq_mode mq_mode = pf->dev_data->dev_conf.rxmode.mq_mode;
7192
7193         /* RSS setup */
7194         if (mq_mode & ETH_MQ_RX_RSS_FLAG)
7195                 ret = i40e_pf_config_rss(pf);
7196         else
7197                 i40e_pf_disable_rss(pf);
7198
7199         return ret;
7200 }
7201
7202 /* Get the symmetric hash enable configurations per port */
7203 static void
7204 i40e_get_symmetric_hash_enable_per_port(struct i40e_hw *hw, uint8_t *enable)
7205 {
7206         uint32_t reg = i40e_read_rx_ctl(hw, I40E_PRTQF_CTL_0);
7207
7208         *enable = reg & I40E_PRTQF_CTL_0_HSYM_ENA_MASK ? 1 : 0;
7209 }
7210
7211 /* Set the symmetric hash enable configurations per port */
7212 static void
7213 i40e_set_symmetric_hash_enable_per_port(struct i40e_hw *hw, uint8_t enable)
7214 {
7215         uint32_t reg = i40e_read_rx_ctl(hw, I40E_PRTQF_CTL_0);
7216
7217         if (enable > 0) {
7218                 if (reg & I40E_PRTQF_CTL_0_HSYM_ENA_MASK) {
7219                         PMD_DRV_LOG(INFO, "Symmetric hash has already "
7220                                                         "been enabled");
7221                         return;
7222                 }
7223                 reg |= I40E_PRTQF_CTL_0_HSYM_ENA_MASK;
7224         } else {
7225                 if (!(reg & I40E_PRTQF_CTL_0_HSYM_ENA_MASK)) {
7226                         PMD_DRV_LOG(INFO, "Symmetric hash has already "
7227                                                         "been disabled");
7228                         return;
7229                 }
7230                 reg &= ~I40E_PRTQF_CTL_0_HSYM_ENA_MASK;
7231         }
7232         i40e_write_rx_ctl(hw, I40E_PRTQF_CTL_0, reg);
7233         I40E_WRITE_FLUSH(hw);
7234 }
7235
7236 /*
7237  * Get global configurations of hash function type and symmetric hash enable
7238  * per flow type (pctype). Note that global configuration means it affects all
7239  * the ports on the same NIC.
7240  */
7241 static int
7242 i40e_get_hash_filter_global_config(struct i40e_hw *hw,
7243                                    struct rte_eth_hash_global_conf *g_cfg)
7244 {
7245         uint32_t reg, mask = I40E_FLOW_TYPES;
7246         uint16_t i;
7247         enum i40e_filter_pctype pctype;
7248
7249         memset(g_cfg, 0, sizeof(*g_cfg));
7250         reg = i40e_read_rx_ctl(hw, I40E_GLQF_CTL);
7251         if (reg & I40E_GLQF_CTL_HTOEP_MASK)
7252                 g_cfg->hash_func = RTE_ETH_HASH_FUNCTION_TOEPLITZ;
7253         else
7254                 g_cfg->hash_func = RTE_ETH_HASH_FUNCTION_SIMPLE_XOR;
7255         PMD_DRV_LOG(DEBUG, "Hash function is %s",
7256                 (reg & I40E_GLQF_CTL_HTOEP_MASK) ? "Toeplitz" : "Simple XOR");
7257
7258         for (i = 0; mask && i < RTE_ETH_FLOW_MAX; i++) {
7259                 if (!(mask & (1UL << i)))
7260                         continue;
7261                 mask &= ~(1UL << i);
7262                 /* Bit set indicats the coresponding flow type is supported */
7263                 g_cfg->valid_bit_mask[0] |= (1UL << i);
7264                 /* if flowtype is invalid, continue */
7265                 if (!I40E_VALID_FLOW(i))
7266                         continue;
7267                 pctype = i40e_flowtype_to_pctype(i);
7268                 reg = i40e_read_rx_ctl(hw, I40E_GLQF_HSYM(pctype));
7269                 if (reg & I40E_GLQF_HSYM_SYMH_ENA_MASK)
7270                         g_cfg->sym_hash_enable_mask[0] |= (1UL << i);
7271         }
7272
7273         return 0;
7274 }
7275
7276 static int
7277 i40e_hash_global_config_check(struct rte_eth_hash_global_conf *g_cfg)
7278 {
7279         uint32_t i;
7280         uint32_t mask0, i40e_mask = I40E_FLOW_TYPES;
7281
7282         if (g_cfg->hash_func != RTE_ETH_HASH_FUNCTION_TOEPLITZ &&
7283                 g_cfg->hash_func != RTE_ETH_HASH_FUNCTION_SIMPLE_XOR &&
7284                 g_cfg->hash_func != RTE_ETH_HASH_FUNCTION_DEFAULT) {
7285                 PMD_DRV_LOG(ERR, "Unsupported hash function type %d",
7286                                                 g_cfg->hash_func);
7287                 return -EINVAL;
7288         }
7289
7290         /*
7291          * As i40e supports less than 32 flow types, only first 32 bits need to
7292          * be checked.
7293          */
7294         mask0 = g_cfg->valid_bit_mask[0];
7295         for (i = 0; i < RTE_SYM_HASH_MASK_ARRAY_SIZE; i++) {
7296                 if (i == 0) {
7297                         /* Check if any unsupported flow type configured */
7298                         if ((mask0 | i40e_mask) ^ i40e_mask)
7299                                 goto mask_err;
7300                 } else {
7301                         if (g_cfg->valid_bit_mask[i])
7302                                 goto mask_err;
7303                 }
7304         }
7305
7306         return 0;
7307
7308 mask_err:
7309         PMD_DRV_LOG(ERR, "i40e unsupported flow type bit(s) configured");
7310
7311         return -EINVAL;
7312 }
7313
7314 /*
7315  * Set global configurations of hash function type and symmetric hash enable
7316  * per flow type (pctype). Note any modifying global configuration will affect
7317  * all the ports on the same NIC.
7318  */
7319 static int
7320 i40e_set_hash_filter_global_config(struct i40e_hw *hw,
7321                                    struct rte_eth_hash_global_conf *g_cfg)
7322 {
7323         struct i40e_pf *pf = &((struct i40e_adapter *)hw->back)->pf;
7324         int ret;
7325         uint16_t i;
7326         uint32_t reg;
7327         uint32_t mask0 = g_cfg->valid_bit_mask[0];
7328         enum i40e_filter_pctype pctype;
7329
7330         if (pf->support_multi_driver) {
7331                 PMD_DRV_LOG(ERR, "Hash global configuration is not supported.");
7332                 return -ENOTSUP;
7333         }
7334
7335         /* Check the input parameters */
7336         ret = i40e_hash_global_config_check(g_cfg);
7337         if (ret < 0)
7338                 return ret;
7339
7340         for (i = 0; mask0 && i < UINT32_BIT; i++) {
7341                 if (!(mask0 & (1UL << i)))
7342                         continue;
7343                 mask0 &= ~(1UL << i);
7344                 /* if flowtype is invalid, continue */
7345                 if (!I40E_VALID_FLOW(i))
7346                         continue;
7347                 pctype = i40e_flowtype_to_pctype(i);
7348                 reg = (g_cfg->sym_hash_enable_mask[0] & (1UL << i)) ?
7349                                 I40E_GLQF_HSYM_SYMH_ENA_MASK : 0;
7350                 if (hw->mac.type == I40E_MAC_X722) {
7351                         if (pctype == I40E_FILTER_PCTYPE_NONF_IPV4_UDP) {
7352                                 i40e_write_global_rx_ctl(hw, I40E_GLQF_HSYM(
7353                                   I40E_FILTER_PCTYPE_NONF_IPV4_UDP), reg);
7354                                 i40e_write_global_rx_ctl(hw, I40E_GLQF_HSYM(
7355                                   I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP),
7356                                   reg);
7357                                 i40e_write_global_rx_ctl(hw, I40E_GLQF_HSYM(
7358                                   I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP),
7359                                   reg);
7360                         } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV4_TCP) {
7361                                 i40e_write_global_rx_ctl(hw, I40E_GLQF_HSYM(
7362                                   I40E_FILTER_PCTYPE_NONF_IPV4_TCP), reg);
7363                                 i40e_write_global_rx_ctl(hw, I40E_GLQF_HSYM(
7364                                   I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK),
7365                                   reg);
7366                         } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV6_UDP) {
7367                                 i40e_write_global_rx_ctl(hw, I40E_GLQF_HSYM(
7368                                   I40E_FILTER_PCTYPE_NONF_IPV6_UDP), reg);
7369                                 i40e_write_global_rx_ctl(hw, I40E_GLQF_HSYM(
7370                                   I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP),
7371                                   reg);
7372                                 i40e_write_global_rx_ctl(hw, I40E_GLQF_HSYM(
7373                                   I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP),
7374                                   reg);
7375                         } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV6_TCP) {
7376                                 i40e_write_global_rx_ctl(hw, I40E_GLQF_HSYM(
7377                                   I40E_FILTER_PCTYPE_NONF_IPV6_TCP), reg);
7378                                 i40e_write_global_rx_ctl(hw, I40E_GLQF_HSYM(
7379                                   I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK),
7380                                   reg);
7381                         } else {
7382                                 i40e_write_global_rx_ctl(hw,
7383                                                          I40E_GLQF_HSYM(pctype),
7384                                                          reg);
7385                         }
7386                 } else {
7387                         i40e_write_global_rx_ctl(hw, I40E_GLQF_HSYM(pctype),
7388                                                  reg);
7389                 }
7390                 i40e_global_cfg_warning(I40E_WARNING_HSYM);
7391         }
7392
7393         reg = i40e_read_rx_ctl(hw, I40E_GLQF_CTL);
7394         if (g_cfg->hash_func == RTE_ETH_HASH_FUNCTION_TOEPLITZ) {
7395                 /* Toeplitz */
7396                 if (reg & I40E_GLQF_CTL_HTOEP_MASK) {
7397                         PMD_DRV_LOG(DEBUG, "Hash function already set to "
7398                                                                 "Toeplitz");
7399                         goto out;
7400                 }
7401                 reg |= I40E_GLQF_CTL_HTOEP_MASK;
7402         } else if (g_cfg->hash_func == RTE_ETH_HASH_FUNCTION_SIMPLE_XOR) {
7403                 /* Simple XOR */
7404                 if (!(reg & I40E_GLQF_CTL_HTOEP_MASK)) {
7405                         PMD_DRV_LOG(DEBUG, "Hash function already set to "
7406                                                         "Simple XOR");
7407                         goto out;
7408                 }
7409                 reg &= ~I40E_GLQF_CTL_HTOEP_MASK;
7410         } else
7411                 /* Use the default, and keep it as it is */
7412                 goto out;
7413
7414         i40e_write_global_rx_ctl(hw, I40E_GLQF_CTL, reg);
7415         i40e_global_cfg_warning(I40E_WARNING_QF_CTL);
7416
7417 out:
7418         I40E_WRITE_FLUSH(hw);
7419
7420         return 0;
7421 }
7422
7423 /**
7424  * Valid input sets for hash and flow director filters per PCTYPE
7425  */
7426 static uint64_t
7427 i40e_get_valid_input_set(enum i40e_filter_pctype pctype,
7428                 enum rte_filter_type filter)
7429 {
7430         uint64_t valid;
7431
7432         static const uint64_t valid_hash_inset_table[] = {
7433                 [I40E_FILTER_PCTYPE_FRAG_IPV4] =
7434                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7435                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7436                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_SRC |
7437                         I40E_INSET_IPV4_DST | I40E_INSET_IPV4_TOS |
7438                         I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
7439                         I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
7440                         I40E_INSET_FLEX_PAYLOAD,
7441                 [I40E_FILTER_PCTYPE_NONF_IPV4_UDP] =
7442                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7443                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7444                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS |
7445                         I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
7446                         I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
7447                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7448                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
7449                         I40E_INSET_FLEX_PAYLOAD,
7450 #ifdef X722_SUPPORT
7451                 [I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP] =
7452                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7453                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7454                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS |
7455                         I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
7456                         I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
7457                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7458                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
7459                         I40E_INSET_FLEX_PAYLOAD,
7460                 [I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP] =
7461                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7462                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7463                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS |
7464                         I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
7465                         I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
7466                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7467                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
7468                         I40E_INSET_FLEX_PAYLOAD,
7469 #endif
7470                 [I40E_FILTER_PCTYPE_NONF_IPV4_TCP] =
7471                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7472                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7473                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS |
7474                         I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
7475                         I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
7476                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7477                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
7478                         I40E_INSET_TCP_FLAGS | I40E_INSET_FLEX_PAYLOAD,
7479 #ifdef X722_SUPPORT
7480                 [I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK] =
7481                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7482                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7483                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS |
7484                         I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
7485                         I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
7486                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7487                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
7488                         I40E_INSET_TCP_FLAGS | I40E_INSET_FLEX_PAYLOAD,
7489 #endif
7490                 [I40E_FILTER_PCTYPE_NONF_IPV4_SCTP] =
7491                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7492                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7493                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS |
7494                         I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
7495                         I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
7496                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7497                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
7498                         I40E_INSET_SCTP_VT | I40E_INSET_FLEX_PAYLOAD,
7499                 [I40E_FILTER_PCTYPE_NONF_IPV4_OTHER] =
7500                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7501                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7502                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS |
7503                         I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
7504                         I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
7505                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7506                         I40E_INSET_FLEX_PAYLOAD,
7507                 [I40E_FILTER_PCTYPE_FRAG_IPV6] =
7508                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7509                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7510                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
7511                         I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
7512                         I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_TUNNEL_DMAC |
7513                         I40E_INSET_TUNNEL_ID | I40E_INSET_IPV6_SRC |
7514                         I40E_INSET_IPV6_DST | I40E_INSET_FLEX_PAYLOAD,
7515                 [I40E_FILTER_PCTYPE_NONF_IPV6_UDP] =
7516                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7517                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7518                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
7519                         I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
7520                         I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC |
7521                         I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT |
7522                         I40E_INSET_DST_PORT | I40E_INSET_FLEX_PAYLOAD,
7523 #ifdef X722_SUPPORT
7524                 [I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP] =
7525                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7526                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7527                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
7528                         I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
7529                         I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC |
7530                         I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT |
7531                         I40E_INSET_DST_PORT | I40E_INSET_TCP_FLAGS |
7532                         I40E_INSET_FLEX_PAYLOAD,
7533                 [I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP] =
7534                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7535                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7536                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
7537                         I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
7538                         I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC |
7539                         I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT |
7540                         I40E_INSET_DST_PORT | I40E_INSET_TCP_FLAGS |
7541                         I40E_INSET_FLEX_PAYLOAD,
7542 #endif
7543                 [I40E_FILTER_PCTYPE_NONF_IPV6_TCP] =
7544                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7545                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7546                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
7547                         I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
7548                         I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC |
7549                         I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT |
7550                         I40E_INSET_DST_PORT | I40E_INSET_TCP_FLAGS |
7551                         I40E_INSET_FLEX_PAYLOAD,
7552 #ifdef X722_SUPPORT
7553                 [I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK] =
7554                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7555                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7556                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
7557                         I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
7558                         I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC |
7559                         I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT |
7560                         I40E_INSET_DST_PORT | I40E_INSET_TCP_FLAGS |
7561                         I40E_INSET_FLEX_PAYLOAD,
7562 #endif
7563                 [I40E_FILTER_PCTYPE_NONF_IPV6_SCTP] =
7564                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7565                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7566                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
7567                         I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
7568                         I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC |
7569                         I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT |
7570                         I40E_INSET_DST_PORT | I40E_INSET_SCTP_VT |
7571                         I40E_INSET_FLEX_PAYLOAD,
7572                 [I40E_FILTER_PCTYPE_NONF_IPV6_OTHER] =
7573                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7574                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7575                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
7576                         I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
7577                         I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC |
7578                         I40E_INSET_IPV6_DST | I40E_INSET_TUNNEL_ID |
7579                         I40E_INSET_FLEX_PAYLOAD,
7580                 [I40E_FILTER_PCTYPE_L2_PAYLOAD] =
7581                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7582                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7583                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_LAST_ETHER_TYPE |
7584                         I40E_INSET_FLEX_PAYLOAD,
7585         };
7586
7587         /**
7588          * Flow director supports only fields defined in
7589          * union rte_eth_fdir_flow.
7590          */
7591         static const uint64_t valid_fdir_inset_table[] = {
7592                 [I40E_FILTER_PCTYPE_FRAG_IPV4] =
7593                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7594                 I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7595                 I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_PROTO |
7596                 I40E_INSET_IPV4_TTL,
7597                 [I40E_FILTER_PCTYPE_NONF_IPV4_UDP] =
7598                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7599                 I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7600                 I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_TTL |
7601                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7602 #ifdef X722_SUPPORT
7603                 [I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP] =
7604                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7605                 I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7606                 I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_TTL |
7607                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7608                 [I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP] =
7609                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7610                 I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7611                 I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_TTL |
7612                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7613 #endif
7614                 [I40E_FILTER_PCTYPE_NONF_IPV4_TCP] =
7615                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7616                 I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7617                 I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_TTL |
7618                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7619 #ifdef X722_SUPPORT
7620                 [I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK] =
7621                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7622                 I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7623                 I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_TTL |
7624                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7625 #endif
7626                 [I40E_FILTER_PCTYPE_NONF_IPV4_SCTP] =
7627                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7628                 I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7629                 I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_TTL |
7630                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
7631                 I40E_INSET_SCTP_VT,
7632                 [I40E_FILTER_PCTYPE_NONF_IPV4_OTHER] =
7633                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7634                 I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7635                 I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_PROTO |
7636                 I40E_INSET_IPV4_TTL,
7637                 [I40E_FILTER_PCTYPE_FRAG_IPV6] =
7638                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7639                 I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7640                 I40E_INSET_IPV6_TC | I40E_INSET_IPV6_NEXT_HDR |
7641                 I40E_INSET_IPV6_HOP_LIMIT,
7642                 [I40E_FILTER_PCTYPE_NONF_IPV6_UDP] =
7643                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7644                 I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7645                 I40E_INSET_IPV6_TC | I40E_INSET_IPV6_HOP_LIMIT |
7646                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7647 #ifdef X722_SUPPORT
7648                 [I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP] =
7649                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7650                 I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7651                 I40E_INSET_IPV6_TC | I40E_INSET_IPV6_HOP_LIMIT |
7652                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7653                 [I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP] =
7654                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7655                 I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7656                 I40E_INSET_IPV6_TC | I40E_INSET_IPV6_HOP_LIMIT |
7657                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7658 #endif
7659                 [I40E_FILTER_PCTYPE_NONF_IPV6_TCP] =
7660                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7661                 I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7662                 I40E_INSET_IPV6_TC | I40E_INSET_IPV6_HOP_LIMIT |
7663                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7664 #ifdef X722_SUPPORT
7665                 [I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK] =
7666                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7667                 I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7668                 I40E_INSET_IPV6_TC | I40E_INSET_IPV6_HOP_LIMIT |
7669                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7670 #endif
7671                 [I40E_FILTER_PCTYPE_NONF_IPV6_SCTP] =
7672                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7673                 I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7674                 I40E_INSET_IPV6_TC | I40E_INSET_IPV6_HOP_LIMIT |
7675                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
7676                 I40E_INSET_SCTP_VT,
7677                 [I40E_FILTER_PCTYPE_NONF_IPV6_OTHER] =
7678                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7679                 I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7680                 I40E_INSET_IPV6_TC | I40E_INSET_IPV6_NEXT_HDR |
7681                 I40E_INSET_IPV6_HOP_LIMIT,
7682                 [I40E_FILTER_PCTYPE_L2_PAYLOAD] =
7683                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7684                 I40E_INSET_LAST_ETHER_TYPE,
7685         };
7686
7687         if (pctype > I40E_FILTER_PCTYPE_L2_PAYLOAD)
7688                 return 0;
7689         if (filter == RTE_ETH_FILTER_HASH)
7690                 valid = valid_hash_inset_table[pctype];
7691         else
7692                 valid = valid_fdir_inset_table[pctype];
7693
7694         return valid;
7695 }
7696
7697 /**
7698  * Validate if the input set is allowed for a specific PCTYPE
7699  */
7700 static int
7701 i40e_validate_input_set(enum i40e_filter_pctype pctype,
7702                 enum rte_filter_type filter, uint64_t inset)
7703 {
7704         uint64_t valid;
7705
7706         valid = i40e_get_valid_input_set(pctype, filter);
7707         if (inset & (~valid))
7708                 return -EINVAL;
7709
7710         return 0;
7711 }
7712
7713 /* default input set fields combination per pctype */
7714 static uint64_t
7715 i40e_get_default_input_set(uint16_t pctype)
7716 {
7717         static const uint64_t default_inset_table[] = {
7718                 [I40E_FILTER_PCTYPE_FRAG_IPV4] =
7719                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST,
7720                 [I40E_FILTER_PCTYPE_NONF_IPV4_UDP] =
7721                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7722                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7723 #ifdef X722_SUPPORT
7724                 [I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP] =
7725                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7726                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7727                 [I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP] =
7728                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7729                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7730 #endif
7731                 [I40E_FILTER_PCTYPE_NONF_IPV4_TCP] =
7732                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7733                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7734 #ifdef X722_SUPPORT
7735                 [I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK] =
7736                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7737                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7738 #endif
7739                 [I40E_FILTER_PCTYPE_NONF_IPV4_SCTP] =
7740                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7741                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
7742                         I40E_INSET_SCTP_VT,
7743                 [I40E_FILTER_PCTYPE_NONF_IPV4_OTHER] =
7744                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST,
7745                 [I40E_FILTER_PCTYPE_FRAG_IPV6] =
7746                         I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST,
7747                 [I40E_FILTER_PCTYPE_NONF_IPV6_UDP] =
7748                         I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7749                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7750 #ifdef X722_SUPPORT
7751                 [I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP] =
7752                         I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7753                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7754                 [I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP] =
7755                         I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7756                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7757 #endif
7758                 [I40E_FILTER_PCTYPE_NONF_IPV6_TCP] =
7759                         I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7760                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7761 #ifdef X722_SUPPORT
7762                 [I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK] =
7763                         I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7764                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7765 #endif
7766                 [I40E_FILTER_PCTYPE_NONF_IPV6_SCTP] =
7767                         I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7768                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
7769                         I40E_INSET_SCTP_VT,
7770                 [I40E_FILTER_PCTYPE_NONF_IPV6_OTHER] =
7771                         I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST,
7772                 [I40E_FILTER_PCTYPE_L2_PAYLOAD] =
7773                         I40E_INSET_LAST_ETHER_TYPE,
7774         };
7775
7776         if (pctype > I40E_FILTER_PCTYPE_L2_PAYLOAD)
7777                 return 0;
7778
7779         return default_inset_table[pctype];
7780 }
7781
7782 /**
7783  * Parse the input set from index to logical bit masks
7784  */
7785 static int
7786 i40e_parse_input_set(uint64_t *inset,
7787                      enum i40e_filter_pctype pctype,
7788                      enum rte_eth_input_set_field *field,
7789                      uint16_t size)
7790 {
7791         uint16_t i, j;
7792         int ret = -EINVAL;
7793
7794         static const struct {
7795                 enum rte_eth_input_set_field field;
7796                 uint64_t inset;
7797         } inset_convert_table[] = {
7798                 {RTE_ETH_INPUT_SET_NONE, I40E_INSET_NONE},
7799                 {RTE_ETH_INPUT_SET_L2_SRC_MAC, I40E_INSET_SMAC},
7800                 {RTE_ETH_INPUT_SET_L2_DST_MAC, I40E_INSET_DMAC},
7801                 {RTE_ETH_INPUT_SET_L2_OUTER_VLAN, I40E_INSET_VLAN_OUTER},
7802                 {RTE_ETH_INPUT_SET_L2_INNER_VLAN, I40E_INSET_VLAN_INNER},
7803                 {RTE_ETH_INPUT_SET_L2_ETHERTYPE, I40E_INSET_LAST_ETHER_TYPE},
7804                 {RTE_ETH_INPUT_SET_L3_SRC_IP4, I40E_INSET_IPV4_SRC},
7805                 {RTE_ETH_INPUT_SET_L3_DST_IP4, I40E_INSET_IPV4_DST},
7806                 {RTE_ETH_INPUT_SET_L3_IP4_TOS, I40E_INSET_IPV4_TOS},
7807                 {RTE_ETH_INPUT_SET_L3_IP4_PROTO, I40E_INSET_IPV4_PROTO},
7808                 {RTE_ETH_INPUT_SET_L3_IP4_TTL, I40E_INSET_IPV4_TTL},
7809                 {RTE_ETH_INPUT_SET_L3_SRC_IP6, I40E_INSET_IPV6_SRC},
7810                 {RTE_ETH_INPUT_SET_L3_DST_IP6, I40E_INSET_IPV6_DST},
7811                 {RTE_ETH_INPUT_SET_L3_IP6_TC, I40E_INSET_IPV6_TC},
7812                 {RTE_ETH_INPUT_SET_L3_IP6_NEXT_HEADER,
7813                         I40E_INSET_IPV6_NEXT_HDR},
7814                 {RTE_ETH_INPUT_SET_L3_IP6_HOP_LIMITS,
7815                         I40E_INSET_IPV6_HOP_LIMIT},
7816                 {RTE_ETH_INPUT_SET_L4_UDP_SRC_PORT, I40E_INSET_SRC_PORT},
7817                 {RTE_ETH_INPUT_SET_L4_TCP_SRC_PORT, I40E_INSET_SRC_PORT},
7818                 {RTE_ETH_INPUT_SET_L4_SCTP_SRC_PORT, I40E_INSET_SRC_PORT},
7819                 {RTE_ETH_INPUT_SET_L4_UDP_DST_PORT, I40E_INSET_DST_PORT},
7820                 {RTE_ETH_INPUT_SET_L4_TCP_DST_PORT, I40E_INSET_DST_PORT},
7821                 {RTE_ETH_INPUT_SET_L4_SCTP_DST_PORT, I40E_INSET_DST_PORT},
7822                 {RTE_ETH_INPUT_SET_L4_SCTP_VERIFICATION_TAG,
7823                         I40E_INSET_SCTP_VT},
7824                 {RTE_ETH_INPUT_SET_TUNNEL_L2_INNER_DST_MAC,
7825                         I40E_INSET_TUNNEL_DMAC},
7826                 {RTE_ETH_INPUT_SET_TUNNEL_L2_INNER_VLAN,
7827                         I40E_INSET_VLAN_TUNNEL},
7828                 {RTE_ETH_INPUT_SET_TUNNEL_L4_UDP_KEY,
7829                         I40E_INSET_TUNNEL_ID},
7830                 {RTE_ETH_INPUT_SET_TUNNEL_GRE_KEY, I40E_INSET_TUNNEL_ID},
7831                 {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_1ST_WORD,
7832                         I40E_INSET_FLEX_PAYLOAD_W1},
7833                 {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_2ND_WORD,
7834                         I40E_INSET_FLEX_PAYLOAD_W2},
7835                 {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_3RD_WORD,
7836                         I40E_INSET_FLEX_PAYLOAD_W3},
7837                 {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_4TH_WORD,
7838                         I40E_INSET_FLEX_PAYLOAD_W4},
7839                 {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_5TH_WORD,
7840                         I40E_INSET_FLEX_PAYLOAD_W5},
7841                 {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_6TH_WORD,
7842                         I40E_INSET_FLEX_PAYLOAD_W6},
7843                 {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_7TH_WORD,
7844                         I40E_INSET_FLEX_PAYLOAD_W7},
7845                 {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_8TH_WORD,
7846                         I40E_INSET_FLEX_PAYLOAD_W8},
7847         };
7848
7849         if (!inset || !field || size > RTE_ETH_INSET_SIZE_MAX)
7850                 return ret;
7851
7852         /* Only one item allowed for default or all */
7853         if (size == 1) {
7854                 if (field[0] == RTE_ETH_INPUT_SET_DEFAULT) {
7855                         *inset = i40e_get_default_input_set(pctype);
7856                         return 0;
7857                 } else if (field[0] == RTE_ETH_INPUT_SET_NONE) {
7858                         *inset = I40E_INSET_NONE;
7859                         return 0;
7860                 }
7861         }
7862
7863         for (i = 0, *inset = 0; i < size; i++) {
7864                 for (j = 0; j < RTE_DIM(inset_convert_table); j++) {
7865                         if (field[i] == inset_convert_table[j].field) {
7866                                 *inset |= inset_convert_table[j].inset;
7867                                 break;
7868                         }
7869                 }
7870
7871                 /* It contains unsupported input set, return immediately */
7872                 if (j == RTE_DIM(inset_convert_table))
7873                         return ret;
7874         }
7875
7876         return 0;
7877 }
7878
7879 /**
7880  * Translate the input set from bit masks to register aware bit masks
7881  * and vice versa
7882  */
7883 static uint64_t
7884 i40e_translate_input_set_reg(enum i40e_mac_type type, uint64_t input)
7885 {
7886         uint64_t val = 0;
7887         uint16_t i;
7888
7889         struct inset_map {
7890                 uint64_t inset;
7891                 uint64_t inset_reg;
7892         };
7893
7894         static const struct inset_map inset_map_common[] = {
7895                 {I40E_INSET_DMAC, I40E_REG_INSET_L2_DMAC},
7896                 {I40E_INSET_SMAC, I40E_REG_INSET_L2_SMAC},
7897                 {I40E_INSET_VLAN_OUTER, I40E_REG_INSET_L2_OUTER_VLAN},
7898                 {I40E_INSET_VLAN_INNER, I40E_REG_INSET_L2_INNER_VLAN},
7899                 {I40E_INSET_LAST_ETHER_TYPE, I40E_REG_INSET_LAST_ETHER_TYPE},
7900                 {I40E_INSET_IPV4_TOS, I40E_REG_INSET_L3_IP4_TOS},
7901                 {I40E_INSET_IPV6_SRC, I40E_REG_INSET_L3_SRC_IP6},
7902                 {I40E_INSET_IPV6_DST, I40E_REG_INSET_L3_DST_IP6},
7903                 {I40E_INSET_IPV6_TC, I40E_REG_INSET_L3_IP6_TC},
7904                 {I40E_INSET_IPV6_NEXT_HDR, I40E_REG_INSET_L3_IP6_NEXT_HDR},
7905                 {I40E_INSET_IPV6_HOP_LIMIT, I40E_REG_INSET_L3_IP6_HOP_LIMIT},
7906                 {I40E_INSET_SRC_PORT, I40E_REG_INSET_L4_SRC_PORT},
7907                 {I40E_INSET_DST_PORT, I40E_REG_INSET_L4_DST_PORT},
7908                 {I40E_INSET_SCTP_VT, I40E_REG_INSET_L4_SCTP_VERIFICATION_TAG},
7909                 {I40E_INSET_TUNNEL_ID, I40E_REG_INSET_TUNNEL_ID},
7910                 {I40E_INSET_TUNNEL_DMAC,
7911                         I40E_REG_INSET_TUNNEL_L2_INNER_DST_MAC},
7912                 {I40E_INSET_TUNNEL_IPV4_DST, I40E_REG_INSET_TUNNEL_L3_DST_IP4},
7913                 {I40E_INSET_TUNNEL_IPV6_DST, I40E_REG_INSET_TUNNEL_L3_DST_IP6},
7914                 {I40E_INSET_TUNNEL_SRC_PORT,
7915                         I40E_REG_INSET_TUNNEL_L4_UDP_SRC_PORT},
7916                 {I40E_INSET_TUNNEL_DST_PORT,
7917                         I40E_REG_INSET_TUNNEL_L4_UDP_DST_PORT},
7918                 {I40E_INSET_VLAN_TUNNEL, I40E_REG_INSET_TUNNEL_VLAN},
7919                 {I40E_INSET_FLEX_PAYLOAD_W1, I40E_REG_INSET_FLEX_PAYLOAD_WORD1},
7920                 {I40E_INSET_FLEX_PAYLOAD_W2, I40E_REG_INSET_FLEX_PAYLOAD_WORD2},
7921                 {I40E_INSET_FLEX_PAYLOAD_W3, I40E_REG_INSET_FLEX_PAYLOAD_WORD3},
7922                 {I40E_INSET_FLEX_PAYLOAD_W4, I40E_REG_INSET_FLEX_PAYLOAD_WORD4},
7923                 {I40E_INSET_FLEX_PAYLOAD_W5, I40E_REG_INSET_FLEX_PAYLOAD_WORD5},
7924                 {I40E_INSET_FLEX_PAYLOAD_W6, I40E_REG_INSET_FLEX_PAYLOAD_WORD6},
7925                 {I40E_INSET_FLEX_PAYLOAD_W7, I40E_REG_INSET_FLEX_PAYLOAD_WORD7},
7926                 {I40E_INSET_FLEX_PAYLOAD_W8, I40E_REG_INSET_FLEX_PAYLOAD_WORD8},
7927         };
7928
7929     /* some different registers map in x722*/
7930         static const struct inset_map inset_map_diff_x722[] = {
7931                 {I40E_INSET_IPV4_SRC, I40E_X722_REG_INSET_L3_SRC_IP4},
7932                 {I40E_INSET_IPV4_DST, I40E_X722_REG_INSET_L3_DST_IP4},
7933                 {I40E_INSET_IPV4_PROTO, I40E_X722_REG_INSET_L3_IP4_PROTO},
7934                 {I40E_INSET_IPV4_TTL, I40E_X722_REG_INSET_L3_IP4_TTL},
7935         };
7936
7937         static const struct inset_map inset_map_diff_not_x722[] = {
7938                 {I40E_INSET_IPV4_SRC, I40E_REG_INSET_L3_SRC_IP4},
7939                 {I40E_INSET_IPV4_DST, I40E_REG_INSET_L3_DST_IP4},
7940                 {I40E_INSET_IPV4_PROTO, I40E_REG_INSET_L3_IP4_PROTO},
7941                 {I40E_INSET_IPV4_TTL, I40E_REG_INSET_L3_IP4_TTL},
7942         };
7943
7944         if (input == 0)
7945                 return val;
7946
7947         /* Translate input set to register aware inset */
7948         if (type == I40E_MAC_X722) {
7949                 for (i = 0; i < RTE_DIM(inset_map_diff_x722); i++) {
7950                         if (input & inset_map_diff_x722[i].inset)
7951                                 val |= inset_map_diff_x722[i].inset_reg;
7952                 }
7953         } else {
7954                 for (i = 0; i < RTE_DIM(inset_map_diff_not_x722); i++) {
7955                         if (input & inset_map_diff_not_x722[i].inset)
7956                                 val |= inset_map_diff_not_x722[i].inset_reg;
7957                 }
7958         }
7959
7960         for (i = 0; i < RTE_DIM(inset_map_common); i++) {
7961                 if (input & inset_map_common[i].inset)
7962                         val |= inset_map_common[i].inset_reg;
7963         }
7964
7965         return val;
7966 }
7967
7968 static int
7969 i40e_generate_inset_mask_reg(uint64_t inset, uint32_t *mask, uint8_t nb_elem)
7970 {
7971         uint8_t i, idx = 0;
7972         uint64_t inset_need_mask = inset;
7973
7974         static const struct {
7975                 uint64_t inset;
7976                 uint32_t mask;
7977         } inset_mask_map[] = {
7978                 {I40E_INSET_IPV4_TOS, I40E_INSET_IPV4_TOS_MASK},
7979                 {I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL, 0},
7980                 {I40E_INSET_IPV4_PROTO, I40E_INSET_IPV4_PROTO_MASK},
7981                 {I40E_INSET_IPV4_TTL, I40E_INSET_IPv4_TTL_MASK},
7982                 {I40E_INSET_IPV6_TC, I40E_INSET_IPV6_TC_MASK},
7983                 {I40E_INSET_IPV6_NEXT_HDR | I40E_INSET_IPV6_HOP_LIMIT, 0},
7984                 {I40E_INSET_IPV6_NEXT_HDR, I40E_INSET_IPV6_NEXT_HDR_MASK},
7985                 {I40E_INSET_IPV6_HOP_LIMIT, I40E_INSET_IPV6_HOP_LIMIT_MASK},
7986         };
7987
7988         if (!inset || !mask || !nb_elem)
7989                 return 0;
7990
7991         for (i = 0, idx = 0; i < RTE_DIM(inset_mask_map); i++) {
7992                 /* Clear the inset bit, if no MASK is required,
7993                  * for example proto + ttl
7994                  */
7995                 if ((inset & inset_mask_map[i].inset) ==
7996                      inset_mask_map[i].inset && inset_mask_map[i].mask == 0)
7997                         inset_need_mask &= ~inset_mask_map[i].inset;
7998                 if (!inset_need_mask)
7999                         return 0;
8000         }
8001         for (i = 0, idx = 0; i < RTE_DIM(inset_mask_map); i++) {
8002                 if ((inset_need_mask & inset_mask_map[i].inset) ==
8003                     inset_mask_map[i].inset) {
8004                         if (idx >= nb_elem) {
8005                                 PMD_DRV_LOG(ERR, "exceed maximal number of bitmasks");
8006                                 return -EINVAL;
8007                         }
8008                         mask[idx] = inset_mask_map[i].mask;
8009                         idx++;
8010                 }
8011         }
8012
8013         return idx;
8014 }
8015
8016 static void
8017 i40e_check_write_reg(struct i40e_hw *hw, uint32_t addr, uint32_t val)
8018 {
8019         uint32_t reg = i40e_read_rx_ctl(hw, addr);
8020
8021         PMD_DRV_LOG(DEBUG, "[0x%08x] original: 0x%08x\n", addr, reg);
8022         if (reg != val)
8023                 i40e_write_rx_ctl(hw, addr, val);
8024         PMD_DRV_LOG(DEBUG, "[0x%08x] after: 0x%08x\n", addr,
8025                     (uint32_t)i40e_read_rx_ctl(hw, addr));
8026 }
8027
8028 static void
8029 i40e_check_write_global_reg(struct i40e_hw *hw, uint32_t addr, uint32_t val)
8030 {
8031         uint32_t reg = i40e_read_rx_ctl(hw, addr);
8032
8033         PMD_DRV_LOG(DEBUG, "[0x%08x] original: 0x%08x", addr, reg);
8034         if (reg != val)
8035                 i40e_write_global_rx_ctl(hw, addr, val);
8036         PMD_DRV_LOG(DEBUG, "[0x%08x] after: 0x%08x", addr,
8037                     (uint32_t)i40e_read_rx_ctl(hw, addr));
8038 }
8039
8040 static void
8041 i40e_filter_input_set_init(struct i40e_pf *pf)
8042 {
8043         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
8044         enum i40e_filter_pctype pctype;
8045         uint64_t input_set, inset_reg;
8046         uint32_t mask_reg[I40E_INSET_MASK_NUM_REG] = {0};
8047         int num, i;
8048
8049         for (pctype = I40E_FILTER_PCTYPE_NONF_IPV4_UDP;
8050              pctype <= I40E_FILTER_PCTYPE_L2_PAYLOAD; pctype++) {
8051                 if (hw->mac.type == I40E_MAC_X722) {
8052                         if (!I40E_VALID_PCTYPE_X722(pctype))
8053                                 continue;
8054                 } else {
8055                         if (!I40E_VALID_PCTYPE(pctype))
8056                                 continue;
8057                 }
8058
8059                 input_set = i40e_get_default_input_set(pctype);
8060
8061                 num = i40e_generate_inset_mask_reg(input_set, mask_reg,
8062                                                    I40E_INSET_MASK_NUM_REG);
8063                 if (num < 0)
8064                         return;
8065
8066                 if (pf->support_multi_driver && num > 0) {
8067                         PMD_DRV_LOG(ERR, "Input set setting is not supported.");
8068                         return;
8069                 }
8070
8071                 inset_reg = i40e_translate_input_set_reg(hw->mac.type,
8072                                         input_set);
8073
8074                 i40e_check_write_reg(hw, I40E_PRTQF_FD_INSET(pctype, 0),
8075                                       (uint32_t)(inset_reg & UINT32_MAX));
8076                 i40e_check_write_reg(hw, I40E_PRTQF_FD_INSET(pctype, 1),
8077                                      (uint32_t)((inset_reg >>
8078                                      I40E_32_BIT_WIDTH) & UINT32_MAX));
8079                 if (!pf->support_multi_driver) {
8080                         i40e_check_write_global_reg(hw,
8081                                             I40E_GLQF_HASH_INSET(0, pctype),
8082                                             (uint32_t)(inset_reg & UINT32_MAX));
8083                         i40e_check_write_global_reg(hw,
8084                                             I40E_GLQF_HASH_INSET(1, pctype),
8085                                             (uint32_t)((inset_reg >>
8086                                             I40E_32_BIT_WIDTH) & UINT32_MAX));
8087
8088                         for (i = 0; i < num; i++) {
8089                                 i40e_check_write_global_reg(hw,
8090                                                     I40E_GLQF_FD_MSK(i, pctype),
8091                                                     mask_reg[i]);
8092                                 i40e_check_write_global_reg(hw,
8093                                                   I40E_GLQF_HASH_MSK(i, pctype),
8094                                                   mask_reg[i]);
8095                         }
8096                         /*clear unused mask registers of the pctype */
8097                         for (i = num; i < I40E_INSET_MASK_NUM_REG; i++) {
8098                                 i40e_check_write_global_reg(hw,
8099                                                     I40E_GLQF_FD_MSK(i, pctype),
8100                                                     0);
8101                                 i40e_check_write_global_reg(hw,
8102                                                   I40E_GLQF_HASH_MSK(i, pctype),
8103                                                     0);
8104                         }
8105                 } else {
8106                         PMD_DRV_LOG(ERR,
8107                                     "Input set setting is not supported.");
8108                 }
8109                 I40E_WRITE_FLUSH(hw);
8110
8111                 /* store the default input set */
8112                 if (!pf->support_multi_driver)
8113                         pf->hash_input_set[pctype] = input_set;
8114                 pf->fdir.input_set[pctype] = input_set;
8115         }
8116
8117         if (!pf->support_multi_driver) {
8118                 i40e_global_cfg_warning(I40E_WARNING_HASH_INSET);
8119                 i40e_global_cfg_warning(I40E_WARNING_FD_MSK);
8120                 i40e_global_cfg_warning(I40E_WARNING_HASH_MSK);
8121         }
8122 }
8123
8124 int
8125 i40e_hash_filter_inset_select(struct i40e_hw *hw,
8126                          struct rte_eth_input_set_conf *conf)
8127 {
8128         struct i40e_pf *pf = &((struct i40e_adapter *)hw->back)->pf;
8129         enum i40e_filter_pctype pctype;
8130         uint64_t input_set, inset_reg = 0;
8131         uint32_t mask_reg[I40E_INSET_MASK_NUM_REG] = {0};
8132         int ret, i, num;
8133
8134         if (pf->support_multi_driver) {
8135                 PMD_DRV_LOG(ERR, "Hash input set setting is not supported.");
8136                 return -ENOTSUP;
8137         }
8138
8139         if (!conf) {
8140                 PMD_DRV_LOG(ERR, "Invalid pointer");
8141                 return -EFAULT;
8142         }
8143         if (conf->op != RTE_ETH_INPUT_SET_SELECT &&
8144             conf->op != RTE_ETH_INPUT_SET_ADD) {
8145                 PMD_DRV_LOG(ERR, "Unsupported input set operation");
8146                 return -EINVAL;
8147         }
8148
8149         if (!I40E_VALID_FLOW(conf->flow_type)) {
8150                 PMD_DRV_LOG(ERR, "invalid flow_type input.");
8151                 return -EINVAL;
8152         }
8153
8154         if (hw->mac.type == I40E_MAC_X722) {
8155                 /* get translated pctype value in fd pctype register */
8156                 pctype = (enum i40e_filter_pctype)i40e_read_rx_ctl(hw,
8157                         I40E_GLQF_FD_PCTYPES((int)i40e_flowtype_to_pctype(
8158                         conf->flow_type)));
8159         } else
8160                 pctype = i40e_flowtype_to_pctype(conf->flow_type);
8161
8162         ret = i40e_parse_input_set(&input_set, pctype, conf->field,
8163                                    conf->inset_size);
8164         if (ret) {
8165                 PMD_DRV_LOG(ERR, "Failed to parse input set");
8166                 return -EINVAL;
8167         }
8168         if (i40e_validate_input_set(pctype, RTE_ETH_FILTER_HASH,
8169                                     input_set) != 0) {
8170                 PMD_DRV_LOG(ERR, "Invalid input set");
8171                 return -EINVAL;
8172         }
8173         if (conf->op == RTE_ETH_INPUT_SET_ADD) {
8174                 /* get inset value in register */
8175                 inset_reg = i40e_read_rx_ctl(hw, I40E_GLQF_HASH_INSET(1, pctype));
8176                 inset_reg <<= I40E_32_BIT_WIDTH;
8177                 inset_reg |= i40e_read_rx_ctl(hw, I40E_GLQF_HASH_INSET(0, pctype));
8178                 input_set |= pf->hash_input_set[pctype];
8179         }
8180         num = i40e_generate_inset_mask_reg(input_set, mask_reg,
8181                                            I40E_INSET_MASK_NUM_REG);
8182         if (num < 0)
8183                 return -EINVAL;
8184
8185         inset_reg |= i40e_translate_input_set_reg(hw->mac.type, input_set);
8186
8187         i40e_check_write_global_reg(hw, I40E_GLQF_HASH_INSET(0, pctype),
8188                                     (uint32_t)(inset_reg & UINT32_MAX));
8189         i40e_check_write_global_reg(hw, I40E_GLQF_HASH_INSET(1, pctype),
8190                                     (uint32_t)((inset_reg >>
8191                                     I40E_32_BIT_WIDTH) & UINT32_MAX));
8192         i40e_global_cfg_warning(I40E_WARNING_HASH_INSET);
8193
8194         for (i = 0; i < num; i++)
8195                 i40e_check_write_global_reg(hw, I40E_GLQF_HASH_MSK(i, pctype),
8196                                             mask_reg[i]);
8197         /*clear unused mask registers of the pctype */
8198         for (i = num; i < I40E_INSET_MASK_NUM_REG; i++)
8199                 i40e_check_write_global_reg(hw, I40E_GLQF_HASH_MSK(i, pctype),
8200                                             0);
8201         i40e_global_cfg_warning(I40E_WARNING_HASH_MSK);
8202         I40E_WRITE_FLUSH(hw);
8203
8204         pf->hash_input_set[pctype] = input_set;
8205         return 0;
8206 }
8207
8208 int
8209 i40e_fdir_filter_inset_select(struct i40e_pf *pf,
8210                          struct rte_eth_input_set_conf *conf)
8211 {
8212         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
8213         enum i40e_filter_pctype pctype;
8214         uint64_t input_set, inset_reg = 0;
8215         uint32_t mask_reg[I40E_INSET_MASK_NUM_REG] = {0};
8216         int ret, i, num;
8217
8218         if (!hw || !conf) {
8219                 PMD_DRV_LOG(ERR, "Invalid pointer");
8220                 return -EFAULT;
8221         }
8222         if (conf->op != RTE_ETH_INPUT_SET_SELECT &&
8223             conf->op != RTE_ETH_INPUT_SET_ADD) {
8224                 PMD_DRV_LOG(ERR, "Unsupported input set operation");
8225                 return -EINVAL;
8226         }
8227
8228         if (!I40E_VALID_FLOW(conf->flow_type)) {
8229                 PMD_DRV_LOG(ERR, "invalid flow_type input.");
8230                 return -EINVAL;
8231         }
8232
8233         pctype = i40e_flowtype_to_pctype(conf->flow_type);
8234
8235         ret = i40e_parse_input_set(&input_set, pctype, conf->field,
8236                                    conf->inset_size);
8237         if (ret) {
8238                 PMD_DRV_LOG(ERR, "Failed to parse input set");
8239                 return -EINVAL;
8240         }
8241         if (i40e_validate_input_set(pctype, RTE_ETH_FILTER_FDIR,
8242                                     input_set) != 0) {
8243                 PMD_DRV_LOG(ERR, "Invalid input set");
8244                 return -EINVAL;
8245         }
8246
8247         /* get inset value in register */
8248         inset_reg = i40e_read_rx_ctl(hw, I40E_PRTQF_FD_INSET(pctype, 1));
8249         inset_reg <<= I40E_32_BIT_WIDTH;
8250         inset_reg |= i40e_read_rx_ctl(hw, I40E_PRTQF_FD_INSET(pctype, 0));
8251
8252         /* Can not change the inset reg for flex payload for fdir,
8253          * it is done by writing I40E_PRTQF_FD_FLXINSET
8254          * in i40e_set_flex_mask_on_pctype.
8255          */
8256         if (conf->op == RTE_ETH_INPUT_SET_SELECT)
8257                 inset_reg &= I40E_REG_INSET_FLEX_PAYLOAD_WORDS;
8258         else
8259                 input_set |= pf->fdir.input_set[pctype];
8260         num = i40e_generate_inset_mask_reg(input_set, mask_reg,
8261                                            I40E_INSET_MASK_NUM_REG);
8262         if (num < 0)
8263                 return -EINVAL;
8264
8265         if (pf->support_multi_driver && num > 0) {
8266                 PMD_DRV_LOG(ERR, "FDIR bit mask is not supported.");
8267                 return -ENOTSUP;
8268         }
8269
8270         inset_reg |= i40e_translate_input_set_reg(hw->mac.type, input_set);
8271
8272         i40e_check_write_reg(hw, I40E_PRTQF_FD_INSET(pctype, 0),
8273                               (uint32_t)(inset_reg & UINT32_MAX));
8274         i40e_check_write_reg(hw, I40E_PRTQF_FD_INSET(pctype, 1),
8275                              (uint32_t)((inset_reg >>
8276                              I40E_32_BIT_WIDTH) & UINT32_MAX));
8277
8278         if (!pf->support_multi_driver) {
8279                 for (i = 0; i < num; i++)
8280                         i40e_check_write_global_reg(hw,
8281                                                     I40E_GLQF_FD_MSK(i, pctype),
8282                                                     mask_reg[i]);
8283                 /*clear unused mask registers of the pctype */
8284                 for (i = num; i < I40E_INSET_MASK_NUM_REG; i++)
8285                         i40e_check_write_global_reg(hw,
8286                                                     I40E_GLQF_FD_MSK(i, pctype),
8287                                                     0);
8288                 i40e_global_cfg_warning(I40E_WARNING_FD_MSK);
8289         } else {
8290                 PMD_DRV_LOG(ERR, "FDIR bit mask is not supported.");
8291         }
8292         I40E_WRITE_FLUSH(hw);
8293
8294         pf->fdir.input_set[pctype] = input_set;
8295         return 0;
8296 }
8297
8298 static int
8299 i40e_hash_filter_get(struct i40e_hw *hw, struct rte_eth_hash_filter_info *info)
8300 {
8301         int ret = 0;
8302
8303         if (!hw || !info) {
8304                 PMD_DRV_LOG(ERR, "Invalid pointer");
8305                 return -EFAULT;
8306         }
8307
8308         switch (info->info_type) {
8309         case RTE_ETH_HASH_FILTER_SYM_HASH_ENA_PER_PORT:
8310                 i40e_get_symmetric_hash_enable_per_port(hw,
8311                                         &(info->info.enable));
8312                 break;
8313         case RTE_ETH_HASH_FILTER_GLOBAL_CONFIG:
8314                 ret = i40e_get_hash_filter_global_config(hw,
8315                                 &(info->info.global_conf));
8316                 break;
8317         default:
8318                 PMD_DRV_LOG(ERR, "Hash filter info type (%d) not supported",
8319                                                         info->info_type);
8320                 ret = -EINVAL;
8321                 break;
8322         }
8323
8324         return ret;
8325 }
8326
8327 static int
8328 i40e_hash_filter_set(struct i40e_hw *hw, struct rte_eth_hash_filter_info *info)
8329 {
8330         int ret = 0;
8331
8332         if (!hw || !info) {
8333                 PMD_DRV_LOG(ERR, "Invalid pointer");
8334                 return -EFAULT;
8335         }
8336
8337         switch (info->info_type) {
8338         case RTE_ETH_HASH_FILTER_SYM_HASH_ENA_PER_PORT:
8339                 i40e_set_symmetric_hash_enable_per_port(hw, info->info.enable);
8340                 break;
8341         case RTE_ETH_HASH_FILTER_GLOBAL_CONFIG:
8342                 ret = i40e_set_hash_filter_global_config(hw,
8343                                 &(info->info.global_conf));
8344                 break;
8345         case RTE_ETH_HASH_FILTER_INPUT_SET_SELECT:
8346                 ret = i40e_hash_filter_inset_select(hw,
8347                                                &(info->info.input_set_conf));
8348                 break;
8349
8350         default:
8351                 PMD_DRV_LOG(ERR, "Hash filter info type (%d) not supported",
8352                                                         info->info_type);
8353                 ret = -EINVAL;
8354                 break;
8355         }
8356
8357         return ret;
8358 }
8359
8360 /* Operations for hash function */
8361 static int
8362 i40e_hash_filter_ctrl(struct rte_eth_dev *dev,
8363                       enum rte_filter_op filter_op,
8364                       void *arg)
8365 {
8366         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8367         int ret = 0;
8368
8369         switch (filter_op) {
8370         case RTE_ETH_FILTER_NOP:
8371                 break;
8372         case RTE_ETH_FILTER_GET:
8373                 ret = i40e_hash_filter_get(hw,
8374                         (struct rte_eth_hash_filter_info *)arg);
8375                 break;
8376         case RTE_ETH_FILTER_SET:
8377                 ret = i40e_hash_filter_set(hw,
8378                         (struct rte_eth_hash_filter_info *)arg);
8379                 break;
8380         default:
8381                 PMD_DRV_LOG(WARNING, "Filter operation (%d) not supported",
8382                                                                 filter_op);
8383                 ret = -ENOTSUP;
8384                 break;
8385         }
8386
8387         return ret;
8388 }
8389
8390 /*
8391  * Configure ethertype filter, which can director packet by filtering
8392  * with mac address and ether_type or only ether_type
8393  */
8394 static int
8395 i40e_ethertype_filter_set(struct i40e_pf *pf,
8396                         struct rte_eth_ethertype_filter *filter,
8397                         bool add)
8398 {
8399         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
8400         struct i40e_control_filter_stats stats;
8401         uint16_t flags = 0;
8402         int ret;
8403
8404         if (filter->queue >= pf->dev_data->nb_rx_queues) {
8405                 PMD_DRV_LOG(ERR, "Invalid queue ID");
8406                 return -EINVAL;
8407         }
8408         if (filter->ether_type == ETHER_TYPE_IPv4 ||
8409                 filter->ether_type == ETHER_TYPE_IPv6) {
8410                 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in"
8411                         " control packet filter.", filter->ether_type);
8412                 return -EINVAL;
8413         }
8414         if (filter->ether_type == ETHER_TYPE_VLAN)
8415                 PMD_DRV_LOG(WARNING, "filter vlan ether_type in first tag is"
8416                         " not supported.");
8417
8418         if (!(filter->flags & RTE_ETHTYPE_FLAGS_MAC))
8419                 flags |= I40E_AQC_ADD_CONTROL_PACKET_FLAGS_IGNORE_MAC;
8420         if (filter->flags & RTE_ETHTYPE_FLAGS_DROP)
8421                 flags |= I40E_AQC_ADD_CONTROL_PACKET_FLAGS_DROP;
8422         flags |= I40E_AQC_ADD_CONTROL_PACKET_FLAGS_TO_QUEUE;
8423
8424         memset(&stats, 0, sizeof(stats));
8425         ret = i40e_aq_add_rem_control_packet_filter(hw,
8426                         filter->mac_addr.addr_bytes,
8427                         filter->ether_type, flags,
8428                         pf->main_vsi->seid,
8429                         filter->queue, add, &stats, NULL);
8430
8431         PMD_DRV_LOG(INFO, "add/rem control packet filter, return %d,"
8432                          " mac_etype_used = %u, etype_used = %u,"
8433                          " mac_etype_free = %u, etype_free = %u\n",
8434                          ret, stats.mac_etype_used, stats.etype_used,
8435                          stats.mac_etype_free, stats.etype_free);
8436         if (ret < 0)
8437                 return -ENOSYS;
8438         return 0;
8439 }
8440
8441 /*
8442  * Handle operations for ethertype filter.
8443  */
8444 static int
8445 i40e_ethertype_filter_handle(struct rte_eth_dev *dev,
8446                                 enum rte_filter_op filter_op,
8447                                 void *arg)
8448 {
8449         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
8450         int ret = 0;
8451
8452         if (filter_op == RTE_ETH_FILTER_NOP)
8453                 return ret;
8454
8455         if (arg == NULL) {
8456                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u",
8457                             filter_op);
8458                 return -EINVAL;
8459         }
8460
8461         switch (filter_op) {
8462         case RTE_ETH_FILTER_ADD:
8463                 ret = i40e_ethertype_filter_set(pf,
8464                         (struct rte_eth_ethertype_filter *)arg,
8465                         TRUE);
8466                 break;
8467         case RTE_ETH_FILTER_DELETE:
8468                 ret = i40e_ethertype_filter_set(pf,
8469                         (struct rte_eth_ethertype_filter *)arg,
8470                         FALSE);
8471                 break;
8472         default:
8473                 PMD_DRV_LOG(ERR, "unsupported operation %u\n", filter_op);
8474                 ret = -ENOSYS;
8475                 break;
8476         }
8477         return ret;
8478 }
8479
8480 static int
8481 i40e_dev_filter_ctrl(struct rte_eth_dev *dev,
8482                      enum rte_filter_type filter_type,
8483                      enum rte_filter_op filter_op,
8484                      void *arg)
8485 {
8486         int ret = 0;
8487
8488         if (dev == NULL)
8489                 return -EINVAL;
8490
8491         switch (filter_type) {
8492         case RTE_ETH_FILTER_NONE:
8493                 /* For global configuration */
8494                 ret = i40e_filter_ctrl_global_config(dev, filter_op, arg);
8495                 break;
8496         case RTE_ETH_FILTER_HASH:
8497                 ret = i40e_hash_filter_ctrl(dev, filter_op, arg);
8498                 break;
8499         case RTE_ETH_FILTER_MACVLAN:
8500                 ret = i40e_mac_filter_handle(dev, filter_op, arg);
8501                 break;
8502         case RTE_ETH_FILTER_ETHERTYPE:
8503                 ret = i40e_ethertype_filter_handle(dev, filter_op, arg);
8504                 break;
8505         case RTE_ETH_FILTER_TUNNEL:
8506                 ret = i40e_tunnel_filter_handle(dev, filter_op, arg);
8507                 break;
8508         case RTE_ETH_FILTER_FDIR:
8509                 ret = i40e_fdir_ctrl_func(dev, filter_op, arg);
8510                 break;
8511         default:
8512                 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported",
8513                                                         filter_type);
8514                 ret = -EINVAL;
8515                 break;
8516         }
8517
8518         return ret;
8519 }
8520
8521 /*
8522  * Check and enable Extended Tag.
8523  * Enabling Extended Tag is important for 40G performance.
8524  */
8525 static void
8526 i40e_enable_extended_tag(struct rte_eth_dev *dev)
8527 {
8528         uint32_t buf = 0;
8529         int ret;
8530
8531         ret = rte_eal_pci_read_config(dev->pci_dev, &buf, sizeof(buf),
8532                                       PCI_DEV_CAP_REG);
8533         if (ret < 0) {
8534                 PMD_DRV_LOG(ERR, "Failed to read PCI offset 0x%x",
8535                             PCI_DEV_CAP_REG);
8536                 return;
8537         }
8538         if (!(buf & PCI_DEV_CAP_EXT_TAG_MASK)) {
8539                 PMD_DRV_LOG(ERR, "Does not support Extended Tag");
8540                 return;
8541         }
8542
8543         buf = 0;
8544         ret = rte_eal_pci_read_config(dev->pci_dev, &buf, sizeof(buf),
8545                                       PCI_DEV_CTRL_REG);
8546         if (ret < 0) {
8547                 PMD_DRV_LOG(ERR, "Failed to read PCI offset 0x%x",
8548                             PCI_DEV_CTRL_REG);
8549                 return;
8550         }
8551         if (buf & PCI_DEV_CTRL_EXT_TAG_MASK) {
8552                 PMD_DRV_LOG(DEBUG, "Extended Tag has already been enabled");
8553                 return;
8554         }
8555         buf |= PCI_DEV_CTRL_EXT_TAG_MASK;
8556         ret = rte_eal_pci_write_config(dev->pci_dev, &buf, sizeof(buf),
8557                                        PCI_DEV_CTRL_REG);
8558         if (ret < 0) {
8559                 PMD_DRV_LOG(ERR, "Failed to write PCI offset 0x%x",
8560                             PCI_DEV_CTRL_REG);
8561                 return;
8562         }
8563 }
8564
8565 /*
8566  * As some registers wouldn't be reset unless a global hardware reset,
8567  * hardware initialization is needed to put those registers into an
8568  * expected initial state.
8569  */
8570 static void
8571 i40e_hw_init(struct rte_eth_dev *dev)
8572 {
8573         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8574
8575         i40e_enable_extended_tag(dev);
8576
8577         /* clear the PF Queue Filter control register */
8578         i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, 0);
8579
8580         /* Disable symmetric hash per port */
8581         i40e_set_symmetric_hash_enable_per_port(hw, 0);
8582 }
8583
8584 enum i40e_filter_pctype
8585 i40e_flowtype_to_pctype(uint16_t flow_type)
8586 {
8587         static const enum i40e_filter_pctype pctype_table[] = {
8588                 [RTE_ETH_FLOW_FRAG_IPV4] = I40E_FILTER_PCTYPE_FRAG_IPV4,
8589                 [RTE_ETH_FLOW_NONFRAG_IPV4_UDP] =
8590                         I40E_FILTER_PCTYPE_NONF_IPV4_UDP,
8591                 [RTE_ETH_FLOW_NONFRAG_IPV4_TCP] =
8592                         I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
8593                 [RTE_ETH_FLOW_NONFRAG_IPV4_SCTP] =
8594                         I40E_FILTER_PCTYPE_NONF_IPV4_SCTP,
8595                 [RTE_ETH_FLOW_NONFRAG_IPV4_OTHER] =
8596                         I40E_FILTER_PCTYPE_NONF_IPV4_OTHER,
8597                 [RTE_ETH_FLOW_FRAG_IPV6] = I40E_FILTER_PCTYPE_FRAG_IPV6,
8598                 [RTE_ETH_FLOW_NONFRAG_IPV6_UDP] =
8599                         I40E_FILTER_PCTYPE_NONF_IPV6_UDP,
8600                 [RTE_ETH_FLOW_NONFRAG_IPV6_TCP] =
8601                         I40E_FILTER_PCTYPE_NONF_IPV6_TCP,
8602                 [RTE_ETH_FLOW_NONFRAG_IPV6_SCTP] =
8603                         I40E_FILTER_PCTYPE_NONF_IPV6_SCTP,
8604                 [RTE_ETH_FLOW_NONFRAG_IPV6_OTHER] =
8605                         I40E_FILTER_PCTYPE_NONF_IPV6_OTHER,
8606                 [RTE_ETH_FLOW_L2_PAYLOAD] = I40E_FILTER_PCTYPE_L2_PAYLOAD,
8607         };
8608
8609         return pctype_table[flow_type];
8610 }
8611
8612 uint16_t
8613 i40e_pctype_to_flowtype(enum i40e_filter_pctype pctype)
8614 {
8615         static const uint16_t flowtype_table[] = {
8616                 [I40E_FILTER_PCTYPE_FRAG_IPV4] = RTE_ETH_FLOW_FRAG_IPV4,
8617                 [I40E_FILTER_PCTYPE_NONF_IPV4_UDP] =
8618                         RTE_ETH_FLOW_NONFRAG_IPV4_UDP,
8619 #ifdef X722_SUPPORT
8620                 [I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP] =
8621                         RTE_ETH_FLOW_NONFRAG_IPV4_UDP,
8622                 [I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP] =
8623                         RTE_ETH_FLOW_NONFRAG_IPV4_UDP,
8624 #endif
8625                 [I40E_FILTER_PCTYPE_NONF_IPV4_TCP] =
8626                         RTE_ETH_FLOW_NONFRAG_IPV4_TCP,
8627 #ifdef X722_SUPPORT
8628                 [I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK] =
8629                         RTE_ETH_FLOW_NONFRAG_IPV4_TCP,
8630 #endif
8631                 [I40E_FILTER_PCTYPE_NONF_IPV4_SCTP] =
8632                         RTE_ETH_FLOW_NONFRAG_IPV4_SCTP,
8633                 [I40E_FILTER_PCTYPE_NONF_IPV4_OTHER] =
8634                         RTE_ETH_FLOW_NONFRAG_IPV4_OTHER,
8635                 [I40E_FILTER_PCTYPE_FRAG_IPV6] = RTE_ETH_FLOW_FRAG_IPV6,
8636                 [I40E_FILTER_PCTYPE_NONF_IPV6_UDP] =
8637                         RTE_ETH_FLOW_NONFRAG_IPV6_UDP,
8638 #ifdef X722_SUPPORT
8639                 [I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP] =
8640                         RTE_ETH_FLOW_NONFRAG_IPV6_UDP,
8641                 [I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP] =
8642                         RTE_ETH_FLOW_NONFRAG_IPV6_UDP,
8643 #endif
8644                 [I40E_FILTER_PCTYPE_NONF_IPV6_TCP] =
8645                         RTE_ETH_FLOW_NONFRAG_IPV6_TCP,
8646 #ifdef X722_SUPPORT
8647                 [I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK] =
8648                         RTE_ETH_FLOW_NONFRAG_IPV6_TCP,
8649 #endif
8650                 [I40E_FILTER_PCTYPE_NONF_IPV6_SCTP] =
8651                         RTE_ETH_FLOW_NONFRAG_IPV6_SCTP,
8652                 [I40E_FILTER_PCTYPE_NONF_IPV6_OTHER] =
8653                         RTE_ETH_FLOW_NONFRAG_IPV6_OTHER,
8654                 [I40E_FILTER_PCTYPE_L2_PAYLOAD] = RTE_ETH_FLOW_L2_PAYLOAD,
8655         };
8656
8657         return flowtype_table[pctype];
8658 }
8659
8660 /*
8661  * On X710, performance number is far from the expectation on recent firmware
8662  * versions; on XL710, performance number is also far from the expectation on
8663  * recent firmware versions, if promiscuous mode is disabled, or promiscuous
8664  * mode is enabled and port MAC address is equal to the packet destination MAC
8665  * address. The fix for this issue may not be integrated in the following
8666  * firmware version. So the workaround in software driver is needed. It needs
8667  * to modify the initial values of 3 internal only registers for both X710 and
8668  * XL710. Note that the values for X710 or XL710 could be different, and the
8669  * workaround can be removed when it is fixed in firmware in the future.
8670  */
8671
8672 /* For both X710 and XL710 */
8673 #define I40E_GL_SWR_PRI_JOIN_MAP_0_VALUE_1      0x10000200
8674 #define I40E_GL_SWR_PRI_JOIN_MAP_0_VALUE_2      0x203F0200
8675 #define I40E_GL_SWR_PRI_JOIN_MAP_0              0x26CE00
8676
8677 #define I40E_GL_SWR_PRI_JOIN_MAP_2_VALUE 0x011f0200
8678 #define I40E_GL_SWR_PRI_JOIN_MAP_2       0x26CE08
8679
8680 /* For X722 */
8681 #define I40E_X722_GL_SWR_PRI_JOIN_MAP_0_VALUE 0x20000200
8682 #define I40E_X722_GL_SWR_PRI_JOIN_MAP_2_VALUE 0x013F0200
8683
8684 /* For X710 */
8685 #define I40E_GL_SWR_PM_UP_THR_EF_VALUE   0x03030303
8686 /* For XL710 */
8687 #define I40E_GL_SWR_PM_UP_THR_SF_VALUE   0x06060606
8688 #define I40E_GL_SWR_PM_UP_THR            0x269FBC
8689
8690 /*
8691  * GL_SWR_PM_UP_THR:
8692  * The value is not impacted from the link speed, its value is set according
8693  * to the total number of ports for a better pipe-monitor configuration.
8694  */
8695 static bool
8696 i40e_get_swr_pm_cfg(struct i40e_hw *hw, uint32_t *value)
8697 {
8698 #define I40E_GL_SWR_PM_EF_DEVICE(dev) \
8699                 .device_id = (dev),   \
8700                 .val = I40E_GL_SWR_PM_UP_THR_EF_VALUE
8701
8702 #define I40E_GL_SWR_PM_SF_DEVICE(dev) \
8703                 .device_id = (dev),   \
8704                 .val = I40E_GL_SWR_PM_UP_THR_SF_VALUE
8705
8706         static const struct {
8707                 uint16_t device_id;
8708                 uint32_t val;
8709         } swr_pm_table[] = {
8710                 { I40E_GL_SWR_PM_EF_DEVICE(I40E_DEV_ID_SFP_XL710) },
8711                 { I40E_GL_SWR_PM_EF_DEVICE(I40E_DEV_ID_KX_C) },
8712                 { I40E_GL_SWR_PM_EF_DEVICE(I40E_DEV_ID_10G_BASE_T) },
8713                 { I40E_GL_SWR_PM_EF_DEVICE(I40E_DEV_ID_10G_BASE_T4) },
8714
8715                 { I40E_GL_SWR_PM_SF_DEVICE(I40E_DEV_ID_KX_B) },
8716                 { I40E_GL_SWR_PM_SF_DEVICE(I40E_DEV_ID_QSFP_A) },
8717                 { I40E_GL_SWR_PM_SF_DEVICE(I40E_DEV_ID_QSFP_B) },
8718                 { I40E_GL_SWR_PM_SF_DEVICE(I40E_DEV_ID_20G_KR2) },
8719                 { I40E_GL_SWR_PM_SF_DEVICE(I40E_DEV_ID_20G_KR2_A) },
8720                 { I40E_GL_SWR_PM_SF_DEVICE(I40E_DEV_ID_25G_B) },
8721                 { I40E_GL_SWR_PM_SF_DEVICE(I40E_DEV_ID_25G_SFP28) },
8722         };
8723         uint32_t i;
8724
8725         if (value == NULL) {
8726                 PMD_DRV_LOG(ERR, "value is NULL");
8727                 return false;
8728         }
8729
8730         for (i = 0; i < RTE_DIM(swr_pm_table); i++) {
8731                 if (hw->device_id == swr_pm_table[i].device_id) {
8732                         *value = swr_pm_table[i].val;
8733
8734                         PMD_DRV_LOG(DEBUG, "Device 0x%x with GL_SWR_PM_UP_THR "
8735                                     "value - 0x%08x",
8736                                     hw->device_id, *value);
8737                         return true;
8738                 }
8739         }
8740
8741         return false;
8742 }
8743
8744 static int
8745 i40e_dev_sync_phy_type(struct i40e_hw *hw)
8746 {
8747         enum i40e_status_code status;
8748         struct i40e_aq_get_phy_abilities_resp phy_ab;
8749         int ret = -ENOTSUP;
8750
8751         status = i40e_aq_get_phy_capabilities(hw, false, true, &phy_ab,
8752                                               NULL);
8753
8754         if (status)
8755                 return ret;
8756
8757         return 0;
8758 }
8759
8760 static void
8761 i40e_configure_registers(struct i40e_hw *hw)
8762 {
8763         static struct {
8764                 uint32_t addr;
8765                 uint64_t val;
8766         } reg_table[] = {
8767                 {I40E_GL_SWR_PRI_JOIN_MAP_0, 0},
8768                 {I40E_GL_SWR_PRI_JOIN_MAP_2, 0},
8769                 {I40E_GL_SWR_PM_UP_THR, 0}, /* Compute value dynamically */
8770         };
8771         uint64_t reg;
8772         uint32_t i;
8773         int ret;
8774
8775         for (i = 0; i < RTE_DIM(reg_table); i++) {
8776                 if (reg_table[i].addr == I40E_GL_SWR_PRI_JOIN_MAP_0) {
8777                         if (hw->mac.type == I40E_MAC_X722) /* For X722 */
8778                                 reg_table[i].val =
8779                                         I40E_X722_GL_SWR_PRI_JOIN_MAP_0_VALUE;
8780                         else /* For X710/XL710/XXV710 */
8781                                 if (hw->aq.fw_maj_ver < 6)
8782                                         reg_table[i].val =
8783                                              I40E_GL_SWR_PRI_JOIN_MAP_0_VALUE_1;
8784                                 else
8785                                         reg_table[i].val =
8786                                              I40E_GL_SWR_PRI_JOIN_MAP_0_VALUE_2;
8787                 }
8788
8789                 if (reg_table[i].addr == I40E_GL_SWR_PRI_JOIN_MAP_2) {
8790                         if (hw->mac.type == I40E_MAC_X722) /* For X722 */
8791                                 reg_table[i].val =
8792                                         I40E_X722_GL_SWR_PRI_JOIN_MAP_2_VALUE;
8793                         else /* For X710/XL710/XXV710 */
8794                                 reg_table[i].val =
8795                                         I40E_GL_SWR_PRI_JOIN_MAP_2_VALUE;
8796                 }
8797
8798                 if (reg_table[i].addr == I40E_GL_SWR_PM_UP_THR) {
8799                         uint32_t cfg_val;
8800
8801                         if (!i40e_get_swr_pm_cfg(hw, &cfg_val)) {
8802                                 PMD_DRV_LOG(DEBUG, "Device 0x%x skips "
8803                                             "GL_SWR_PM_UP_THR value fixup",
8804                                             hw->device_id);
8805                                 continue;
8806                         }
8807
8808                         reg_table[i].val = cfg_val;
8809                 }
8810
8811                 ret = i40e_aq_debug_read_register(hw, reg_table[i].addr,
8812                                                         &reg, NULL);
8813                 if (ret < 0) {
8814                         PMD_DRV_LOG(ERR, "Failed to read from 0x%"PRIx32,
8815                                                         reg_table[i].addr);
8816                         break;
8817                 }
8818                 PMD_DRV_LOG(DEBUG, "Read from 0x%"PRIx32": 0x%"PRIx64,
8819                                                 reg_table[i].addr, reg);
8820                 if (reg == reg_table[i].val)
8821                         continue;
8822
8823                 ret = i40e_aq_debug_write_register(hw, reg_table[i].addr,
8824                                                 reg_table[i].val, NULL);
8825                 if (ret < 0) {
8826                         PMD_DRV_LOG(ERR, "Failed to write 0x%"PRIx64" to the "
8827                                 "address of 0x%"PRIx32, reg_table[i].val,
8828                                                         reg_table[i].addr);
8829                         break;
8830                 }
8831                 PMD_DRV_LOG(DEBUG, "Write 0x%"PRIx64" to the address of "
8832                         "0x%"PRIx32, reg_table[i].val, reg_table[i].addr);
8833         }
8834 }
8835
8836 #define I40E_VSI_TSR(_i)            (0x00050800 + ((_i) * 4))
8837 #define I40E_VSI_TSR_QINQ_CONFIG    0xc030
8838 #define I40E_VSI_L2TAGSTXVALID(_i)  (0x00042800 + ((_i) * 4))
8839 #define I40E_VSI_L2TAGSTXVALID_QINQ 0xab
8840 static int
8841 i40e_config_qinq(struct i40e_hw *hw, struct i40e_vsi *vsi)
8842 {
8843         uint32_t reg;
8844         int ret;
8845
8846         if (vsi->vsi_id >= I40E_MAX_NUM_VSIS) {
8847                 PMD_DRV_LOG(ERR, "VSI ID exceeds the maximum");
8848                 return -EINVAL;
8849         }
8850
8851         /* Configure for double VLAN RX stripping */
8852         reg = I40E_READ_REG(hw, I40E_VSI_TSR(vsi->vsi_id));
8853         if ((reg & I40E_VSI_TSR_QINQ_CONFIG) != I40E_VSI_TSR_QINQ_CONFIG) {
8854                 reg |= I40E_VSI_TSR_QINQ_CONFIG;
8855                 ret = i40e_aq_debug_write_register(hw,
8856                                                    I40E_VSI_TSR(vsi->vsi_id),
8857                                                    reg, NULL);
8858                 if (ret < 0) {
8859                         PMD_DRV_LOG(ERR, "Failed to update VSI_TSR[%d]",
8860                                     vsi->vsi_id);
8861                         return I40E_ERR_CONFIG;
8862                 }
8863         }
8864
8865         /* Configure for double VLAN TX insertion */
8866         reg = I40E_READ_REG(hw, I40E_VSI_L2TAGSTXVALID(vsi->vsi_id));
8867         if ((reg & 0xff) != I40E_VSI_L2TAGSTXVALID_QINQ) {
8868                 reg = I40E_VSI_L2TAGSTXVALID_QINQ;
8869                 ret = i40e_aq_debug_write_register(hw,
8870                                                    I40E_VSI_L2TAGSTXVALID(
8871                                                    vsi->vsi_id), reg, NULL);
8872                 if (ret < 0) {
8873                         PMD_DRV_LOG(ERR, "Failed to update "
8874                                 "VSI_L2TAGSTXVALID[%d]", vsi->vsi_id);
8875                         return I40E_ERR_CONFIG;
8876                 }
8877         }
8878
8879         return 0;
8880 }
8881
8882 /**
8883  * i40e_aq_add_mirror_rule
8884  * @hw: pointer to the hardware structure
8885  * @seid: VEB seid to add mirror rule to
8886  * @dst_id: destination vsi seid
8887  * @entries: Buffer which contains the entities to be mirrored
8888  * @count: number of entities contained in the buffer
8889  * @rule_id:the rule_id of the rule to be added
8890  *
8891  * Add a mirror rule for a given veb.
8892  *
8893  **/
8894 static enum i40e_status_code
8895 i40e_aq_add_mirror_rule(struct i40e_hw *hw,
8896                         uint16_t seid, uint16_t dst_id,
8897                         uint16_t rule_type, uint16_t *entries,
8898                         uint16_t count, uint16_t *rule_id)
8899 {
8900         struct i40e_aq_desc desc;
8901         struct i40e_aqc_add_delete_mirror_rule cmd;
8902         struct i40e_aqc_add_delete_mirror_rule_completion *resp =
8903                 (struct i40e_aqc_add_delete_mirror_rule_completion *)
8904                 &desc.params.raw;
8905         uint16_t buff_len;
8906         enum i40e_status_code status;
8907
8908         i40e_fill_default_direct_cmd_desc(&desc,
8909                                           i40e_aqc_opc_add_mirror_rule);
8910         memset(&cmd, 0, sizeof(cmd));
8911
8912         buff_len = sizeof(uint16_t) * count;
8913         desc.datalen = rte_cpu_to_le_16(buff_len);
8914         if (buff_len > 0)
8915                 desc.flags |= rte_cpu_to_le_16(
8916                         (uint16_t)(I40E_AQ_FLAG_BUF | I40E_AQ_FLAG_RD));
8917         cmd.rule_type = rte_cpu_to_le_16(rule_type <<
8918                                 I40E_AQC_MIRROR_RULE_TYPE_SHIFT);
8919         cmd.num_entries = rte_cpu_to_le_16(count);
8920         cmd.seid = rte_cpu_to_le_16(seid);
8921         cmd.destination = rte_cpu_to_le_16(dst_id);
8922
8923         rte_memcpy(&desc.params.raw, &cmd, sizeof(cmd));
8924         status = i40e_asq_send_command(hw, &desc, entries, buff_len, NULL);
8925         PMD_DRV_LOG(INFO, "i40e_aq_add_mirror_rule, aq_status %d,"
8926                          "rule_id = %u"
8927                          " mirror_rules_used = %u, mirror_rules_free = %u,",
8928                          hw->aq.asq_last_status, resp->rule_id,
8929                          resp->mirror_rules_used, resp->mirror_rules_free);
8930         *rule_id = rte_le_to_cpu_16(resp->rule_id);
8931
8932         return status;
8933 }
8934
8935 /**
8936  * i40e_aq_del_mirror_rule
8937  * @hw: pointer to the hardware structure
8938  * @seid: VEB seid to add mirror rule to
8939  * @entries: Buffer which contains the entities to be mirrored
8940  * @count: number of entities contained in the buffer
8941  * @rule_id:the rule_id of the rule to be delete
8942  *
8943  * Delete a mirror rule for a given veb.
8944  *
8945  **/
8946 static enum i40e_status_code
8947 i40e_aq_del_mirror_rule(struct i40e_hw *hw,
8948                 uint16_t seid, uint16_t rule_type, uint16_t *entries,
8949                 uint16_t count, uint16_t rule_id)
8950 {
8951         struct i40e_aq_desc desc;
8952         struct i40e_aqc_add_delete_mirror_rule cmd;
8953         uint16_t buff_len = 0;
8954         enum i40e_status_code status;
8955         void *buff = NULL;
8956
8957         i40e_fill_default_direct_cmd_desc(&desc,
8958                                           i40e_aqc_opc_delete_mirror_rule);
8959         memset(&cmd, 0, sizeof(cmd));
8960         if (rule_type == I40E_AQC_MIRROR_RULE_TYPE_VLAN) {
8961                 desc.flags |= rte_cpu_to_le_16((uint16_t)(I40E_AQ_FLAG_BUF |
8962                                                           I40E_AQ_FLAG_RD));
8963                 cmd.num_entries = count;
8964                 buff_len = sizeof(uint16_t) * count;
8965                 desc.datalen = rte_cpu_to_le_16(buff_len);
8966                 buff = (void *)entries;
8967         } else
8968                 /* rule id is filled in destination field for deleting mirror rule */
8969                 cmd.destination = rte_cpu_to_le_16(rule_id);
8970
8971         cmd.rule_type = rte_cpu_to_le_16(rule_type <<
8972                                 I40E_AQC_MIRROR_RULE_TYPE_SHIFT);
8973         cmd.seid = rte_cpu_to_le_16(seid);
8974
8975         rte_memcpy(&desc.params.raw, &cmd, sizeof(cmd));
8976         status = i40e_asq_send_command(hw, &desc, buff, buff_len, NULL);
8977
8978         return status;
8979 }
8980
8981 /**
8982  * i40e_mirror_rule_set
8983  * @dev: pointer to the hardware structure
8984  * @mirror_conf: mirror rule info
8985  * @sw_id: mirror rule's sw_id
8986  * @on: enable/disable
8987  *
8988  * set a mirror rule.
8989  *
8990  **/
8991 static int
8992 i40e_mirror_rule_set(struct rte_eth_dev *dev,
8993                         struct rte_eth_mirror_conf *mirror_conf,
8994                         uint8_t sw_id, uint8_t on)
8995 {
8996         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
8997         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8998         struct i40e_mirror_rule *it, *mirr_rule = NULL;
8999         struct i40e_mirror_rule *parent = NULL;
9000         uint16_t seid, dst_seid, rule_id;
9001         uint16_t i, j = 0;
9002         int ret;
9003
9004         PMD_DRV_LOG(DEBUG, "i40e_mirror_rule_set: sw_id = %d.", sw_id);
9005
9006         if (pf->main_vsi->veb == NULL || pf->vfs == NULL) {
9007                 PMD_DRV_LOG(ERR, "mirror rule can not be configured"
9008                         " without veb or vfs.");
9009                 return -ENOSYS;
9010         }
9011         if (pf->nb_mirror_rule > I40E_MAX_MIRROR_RULES) {
9012                 PMD_DRV_LOG(ERR, "mirror table is full.");
9013                 return -ENOSPC;
9014         }
9015         if (mirror_conf->dst_pool > pf->vf_num) {
9016                 PMD_DRV_LOG(ERR, "invalid destination pool %u.",
9017                                  mirror_conf->dst_pool);
9018                 return -EINVAL;
9019         }
9020
9021         seid = pf->main_vsi->veb->seid;
9022
9023         TAILQ_FOREACH(it, &pf->mirror_list, rules) {
9024                 if (sw_id <= it->index) {
9025                         mirr_rule = it;
9026                         break;
9027                 }
9028                 parent = it;
9029         }
9030         if (mirr_rule && sw_id == mirr_rule->index) {
9031                 if (on) {
9032                         PMD_DRV_LOG(ERR, "mirror rule exists.");
9033                         return -EEXIST;
9034                 } else {
9035                         ret = i40e_aq_del_mirror_rule(hw, seid,
9036                                         mirr_rule->rule_type,
9037                                         mirr_rule->entries,
9038                                         mirr_rule->num_entries, mirr_rule->id);
9039                         if (ret < 0) {
9040                                 PMD_DRV_LOG(ERR, "failed to remove mirror rule:"
9041                                                    " ret = %d, aq_err = %d.",
9042                                                    ret, hw->aq.asq_last_status);
9043                                 return -ENOSYS;
9044                         }
9045                         TAILQ_REMOVE(&pf->mirror_list, mirr_rule, rules);
9046                         rte_free(mirr_rule);
9047                         pf->nb_mirror_rule--;
9048                         return 0;
9049                 }
9050         } else if (!on) {
9051                 PMD_DRV_LOG(ERR, "mirror rule doesn't exist.");
9052                 return -ENOENT;
9053         }
9054
9055         mirr_rule = rte_zmalloc("i40e_mirror_rule",
9056                                 sizeof(struct i40e_mirror_rule) , 0);
9057         if (!mirr_rule) {
9058                 PMD_DRV_LOG(ERR, "failed to allocate memory");
9059                 return I40E_ERR_NO_MEMORY;
9060         }
9061         switch (mirror_conf->rule_type) {
9062         case ETH_MIRROR_VLAN:
9063                 for (i = 0, j = 0; i < ETH_MIRROR_MAX_VLANS; i++) {
9064                         if (mirror_conf->vlan.vlan_mask & (1ULL << i)) {
9065                                 mirr_rule->entries[j] =
9066                                         mirror_conf->vlan.vlan_id[i];
9067                                 j++;
9068                         }
9069                 }
9070                 if (j == 0) {
9071                         PMD_DRV_LOG(ERR, "vlan is not specified.");
9072                         rte_free(mirr_rule);
9073                         return -EINVAL;
9074                 }
9075                 mirr_rule->rule_type = I40E_AQC_MIRROR_RULE_TYPE_VLAN;
9076                 break;
9077         case ETH_MIRROR_VIRTUAL_POOL_UP:
9078         case ETH_MIRROR_VIRTUAL_POOL_DOWN:
9079                 /* check if the specified pool bit is out of range */
9080                 if (mirror_conf->pool_mask > (uint64_t)(1ULL << (pf->vf_num + 1))) {
9081                         PMD_DRV_LOG(ERR, "pool mask is out of range.");
9082                         rte_free(mirr_rule);
9083                         return -EINVAL;
9084                 }
9085                 for (i = 0, j = 0; i < pf->vf_num; i++) {
9086                         if (mirror_conf->pool_mask & (1ULL << i)) {
9087                                 mirr_rule->entries[j] = pf->vfs[i].vsi->seid;
9088                                 j++;
9089                         }
9090                 }
9091                 if (mirror_conf->pool_mask & (1ULL << pf->vf_num)) {
9092                         /* add pf vsi to entries */
9093                         mirr_rule->entries[j] = pf->main_vsi_seid;
9094                         j++;
9095                 }
9096                 if (j == 0) {
9097                         PMD_DRV_LOG(ERR, "pool is not specified.");
9098                         rte_free(mirr_rule);
9099                         return -EINVAL;
9100                 }
9101                 /* egress and ingress in aq commands means from switch but not port */
9102                 mirr_rule->rule_type =
9103                         (mirror_conf->rule_type == ETH_MIRROR_VIRTUAL_POOL_UP) ?
9104                         I40E_AQC_MIRROR_RULE_TYPE_VPORT_EGRESS :
9105                         I40E_AQC_MIRROR_RULE_TYPE_VPORT_INGRESS;
9106                 break;
9107         case ETH_MIRROR_UPLINK_PORT:
9108                 /* egress and ingress in aq commands means from switch but not port*/
9109                 mirr_rule->rule_type = I40E_AQC_MIRROR_RULE_TYPE_ALL_EGRESS;
9110                 break;
9111         case ETH_MIRROR_DOWNLINK_PORT:
9112                 mirr_rule->rule_type = I40E_AQC_MIRROR_RULE_TYPE_ALL_INGRESS;
9113                 break;
9114         default:
9115                 PMD_DRV_LOG(ERR, "unsupported mirror type %d.",
9116                         mirror_conf->rule_type);
9117                 rte_free(mirr_rule);
9118                 return -EINVAL;
9119         }
9120
9121         /* If the dst_pool is equal to vf_num, consider it as PF */
9122         if (mirror_conf->dst_pool == pf->vf_num)
9123                 dst_seid = pf->main_vsi_seid;
9124         else
9125                 dst_seid = pf->vfs[mirror_conf->dst_pool].vsi->seid;
9126
9127         ret = i40e_aq_add_mirror_rule(hw, seid, dst_seid,
9128                                       mirr_rule->rule_type, mirr_rule->entries,
9129                                       j, &rule_id);
9130         if (ret < 0) {
9131                 PMD_DRV_LOG(ERR, "failed to add mirror rule:"
9132                                    " ret = %d, aq_err = %d.",
9133                                    ret, hw->aq.asq_last_status);
9134                 rte_free(mirr_rule);
9135                 return -ENOSYS;
9136         }
9137
9138         mirr_rule->index = sw_id;
9139         mirr_rule->num_entries = j;
9140         mirr_rule->id = rule_id;
9141         mirr_rule->dst_vsi_seid = dst_seid;
9142
9143         if (parent)
9144                 TAILQ_INSERT_AFTER(&pf->mirror_list, parent, mirr_rule, rules);
9145         else
9146                 TAILQ_INSERT_HEAD(&pf->mirror_list, mirr_rule, rules);
9147
9148         pf->nb_mirror_rule++;
9149         return 0;
9150 }
9151
9152 /**
9153  * i40e_mirror_rule_reset
9154  * @dev: pointer to the device
9155  * @sw_id: mirror rule's sw_id
9156  *
9157  * reset a mirror rule.
9158  *
9159  **/
9160 static int
9161 i40e_mirror_rule_reset(struct rte_eth_dev *dev, uint8_t sw_id)
9162 {
9163         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
9164         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
9165         struct i40e_mirror_rule *it, *mirr_rule = NULL;
9166         uint16_t seid;
9167         int ret;
9168
9169         PMD_DRV_LOG(DEBUG, "i40e_mirror_rule_reset: sw_id = %d.", sw_id);
9170
9171         seid = pf->main_vsi->veb->seid;
9172
9173         TAILQ_FOREACH(it, &pf->mirror_list, rules) {
9174                 if (sw_id == it->index) {
9175                         mirr_rule = it;
9176                         break;
9177                 }
9178         }
9179         if (mirr_rule) {
9180                 ret = i40e_aq_del_mirror_rule(hw, seid,
9181                                 mirr_rule->rule_type,
9182                                 mirr_rule->entries,
9183                                 mirr_rule->num_entries, mirr_rule->id);
9184                 if (ret < 0) {
9185                         PMD_DRV_LOG(ERR, "failed to remove mirror rule:"
9186                                            " status = %d, aq_err = %d.",
9187                                            ret, hw->aq.asq_last_status);
9188                         return -ENOSYS;
9189                 }
9190                 TAILQ_REMOVE(&pf->mirror_list, mirr_rule, rules);
9191                 rte_free(mirr_rule);
9192                 pf->nb_mirror_rule--;
9193         } else {
9194                 PMD_DRV_LOG(ERR, "mirror rule doesn't exist.");
9195                 return -ENOENT;
9196         }
9197         return 0;
9198 }
9199
9200 static uint64_t
9201 i40e_read_systime_cyclecounter(struct rte_eth_dev *dev)
9202 {
9203         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
9204         uint64_t systim_cycles;
9205
9206         systim_cycles = (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_TIME_L);
9207         systim_cycles |= (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_TIME_H)
9208                         << 32;
9209
9210         return systim_cycles;
9211 }
9212
9213 static uint64_t
9214 i40e_read_rx_tstamp_cyclecounter(struct rte_eth_dev *dev, uint8_t index)
9215 {
9216         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
9217         uint64_t rx_tstamp;
9218
9219         rx_tstamp = (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_L(index));
9220         rx_tstamp |= (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_H(index))
9221                         << 32;
9222
9223         return rx_tstamp;
9224 }
9225
9226 static uint64_t
9227 i40e_read_tx_tstamp_cyclecounter(struct rte_eth_dev *dev)
9228 {
9229         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
9230         uint64_t tx_tstamp;
9231
9232         tx_tstamp = (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_TXTIME_L);
9233         tx_tstamp |= (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_TXTIME_H)
9234                         << 32;
9235
9236         return tx_tstamp;
9237 }
9238
9239 static void
9240 i40e_start_timecounters(struct rte_eth_dev *dev)
9241 {
9242         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
9243         struct i40e_adapter *adapter =
9244                         (struct i40e_adapter *)dev->data->dev_private;
9245         struct rte_eth_link link;
9246         uint32_t tsync_inc_l;
9247         uint32_t tsync_inc_h;
9248
9249         /* Get current link speed. */
9250         memset(&link, 0, sizeof(link));
9251         i40e_dev_link_update(dev, 1);
9252         rte_i40e_dev_atomic_read_link_status(dev, &link);
9253
9254         switch (link.link_speed) {
9255         case ETH_SPEED_NUM_40G:
9256                 tsync_inc_l = I40E_PTP_40GB_INCVAL & 0xFFFFFFFF;
9257                 tsync_inc_h = I40E_PTP_40GB_INCVAL >> 32;
9258                 break;
9259         case ETH_SPEED_NUM_10G:
9260                 tsync_inc_l = I40E_PTP_10GB_INCVAL & 0xFFFFFFFF;
9261                 tsync_inc_h = I40E_PTP_10GB_INCVAL >> 32;
9262                 break;
9263         case ETH_SPEED_NUM_1G:
9264                 tsync_inc_l = I40E_PTP_1GB_INCVAL & 0xFFFFFFFF;
9265                 tsync_inc_h = I40E_PTP_1GB_INCVAL >> 32;
9266                 break;
9267         default:
9268                 tsync_inc_l = 0x0;
9269                 tsync_inc_h = 0x0;
9270         }
9271
9272         /* Set the timesync increment value. */
9273         I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_L, tsync_inc_l);
9274         I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_H, tsync_inc_h);
9275
9276         memset(&adapter->systime_tc, 0, sizeof(struct rte_timecounter));
9277         memset(&adapter->rx_tstamp_tc, 0, sizeof(struct rte_timecounter));
9278         memset(&adapter->tx_tstamp_tc, 0, sizeof(struct rte_timecounter));
9279
9280         adapter->systime_tc.cc_mask = I40E_CYCLECOUNTER_MASK;
9281         adapter->systime_tc.cc_shift = 0;
9282         adapter->systime_tc.nsec_mask = 0;
9283
9284         adapter->rx_tstamp_tc.cc_mask = I40E_CYCLECOUNTER_MASK;
9285         adapter->rx_tstamp_tc.cc_shift = 0;
9286         adapter->rx_tstamp_tc.nsec_mask = 0;
9287
9288         adapter->tx_tstamp_tc.cc_mask = I40E_CYCLECOUNTER_MASK;
9289         adapter->tx_tstamp_tc.cc_shift = 0;
9290         adapter->tx_tstamp_tc.nsec_mask = 0;
9291 }
9292
9293 static int
9294 i40e_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta)
9295 {
9296         struct i40e_adapter *adapter =
9297                         (struct i40e_adapter *)dev->data->dev_private;
9298
9299         adapter->systime_tc.nsec += delta;
9300         adapter->rx_tstamp_tc.nsec += delta;
9301         adapter->tx_tstamp_tc.nsec += delta;
9302
9303         return 0;
9304 }
9305
9306 static int
9307 i40e_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts)
9308 {
9309         uint64_t ns;
9310         struct i40e_adapter *adapter =
9311                         (struct i40e_adapter *)dev->data->dev_private;
9312
9313         ns = rte_timespec_to_ns(ts);
9314
9315         /* Set the timecounters to a new value. */
9316         adapter->systime_tc.nsec = ns;
9317         adapter->rx_tstamp_tc.nsec = ns;
9318         adapter->tx_tstamp_tc.nsec = ns;
9319
9320         return 0;
9321 }
9322
9323 static int
9324 i40e_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts)
9325 {
9326         uint64_t ns, systime_cycles;
9327         struct i40e_adapter *adapter =
9328                         (struct i40e_adapter *)dev->data->dev_private;
9329
9330         systime_cycles = i40e_read_systime_cyclecounter(dev);
9331         ns = rte_timecounter_update(&adapter->systime_tc, systime_cycles);
9332         *ts = rte_ns_to_timespec(ns);
9333
9334         return 0;
9335 }
9336
9337 static int
9338 i40e_timesync_enable(struct rte_eth_dev *dev)
9339 {
9340         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
9341         uint32_t tsync_ctl_l;
9342         uint32_t tsync_ctl_h;
9343
9344         /* Stop the timesync system time. */
9345         I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_L, 0x0);
9346         I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_H, 0x0);
9347         /* Reset the timesync system time value. */
9348         I40E_WRITE_REG(hw, I40E_PRTTSYN_TIME_L, 0x0);
9349         I40E_WRITE_REG(hw, I40E_PRTTSYN_TIME_H, 0x0);
9350
9351         i40e_start_timecounters(dev);
9352
9353         /* Clear timesync registers. */
9354         I40E_READ_REG(hw, I40E_PRTTSYN_STAT_0);
9355         I40E_READ_REG(hw, I40E_PRTTSYN_TXTIME_H);
9356         I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_H(0));
9357         I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_H(1));
9358         I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_H(2));
9359         I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_H(3));
9360
9361         /* Enable timestamping of PTP packets. */
9362         tsync_ctl_l = I40E_READ_REG(hw, I40E_PRTTSYN_CTL0);
9363         tsync_ctl_l |= I40E_PRTTSYN_TSYNENA;
9364
9365         tsync_ctl_h = I40E_READ_REG(hw, I40E_PRTTSYN_CTL1);
9366         tsync_ctl_h |= I40E_PRTTSYN_TSYNENA;
9367         tsync_ctl_h |= I40E_PRTTSYN_TSYNTYPE;
9368
9369         I40E_WRITE_REG(hw, I40E_PRTTSYN_CTL0, tsync_ctl_l);
9370         I40E_WRITE_REG(hw, I40E_PRTTSYN_CTL1, tsync_ctl_h);
9371
9372         return 0;
9373 }
9374
9375 static int
9376 i40e_timesync_disable(struct rte_eth_dev *dev)
9377 {
9378         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
9379         uint32_t tsync_ctl_l;
9380         uint32_t tsync_ctl_h;
9381
9382         /* Disable timestamping of transmitted PTP packets. */
9383         tsync_ctl_l = I40E_READ_REG(hw, I40E_PRTTSYN_CTL0);
9384         tsync_ctl_l &= ~I40E_PRTTSYN_TSYNENA;
9385
9386         tsync_ctl_h = I40E_READ_REG(hw, I40E_PRTTSYN_CTL1);
9387         tsync_ctl_h &= ~I40E_PRTTSYN_TSYNENA;
9388
9389         I40E_WRITE_REG(hw, I40E_PRTTSYN_CTL0, tsync_ctl_l);
9390         I40E_WRITE_REG(hw, I40E_PRTTSYN_CTL1, tsync_ctl_h);
9391
9392         /* Reset the timesync increment value. */
9393         I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_L, 0x0);
9394         I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_H, 0x0);
9395
9396         return 0;
9397 }
9398
9399 static int
9400 i40e_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
9401                                 struct timespec *timestamp, uint32_t flags)
9402 {
9403         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
9404         struct i40e_adapter *adapter =
9405                 (struct i40e_adapter *)dev->data->dev_private;
9406
9407         uint32_t sync_status;
9408         uint32_t index = flags & 0x03;
9409         uint64_t rx_tstamp_cycles;
9410         uint64_t ns;
9411
9412         sync_status = I40E_READ_REG(hw, I40E_PRTTSYN_STAT_1);
9413         if ((sync_status & (1 << index)) == 0)
9414                 return -EINVAL;
9415
9416         rx_tstamp_cycles = i40e_read_rx_tstamp_cyclecounter(dev, index);
9417         ns = rte_timecounter_update(&adapter->rx_tstamp_tc, rx_tstamp_cycles);
9418         *timestamp = rte_ns_to_timespec(ns);
9419
9420         return 0;
9421 }
9422
9423 static int
9424 i40e_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
9425                                 struct timespec *timestamp)
9426 {
9427         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
9428         struct i40e_adapter *adapter =
9429                 (struct i40e_adapter *)dev->data->dev_private;
9430
9431         uint32_t sync_status;
9432         uint64_t tx_tstamp_cycles;
9433         uint64_t ns;
9434
9435         sync_status = I40E_READ_REG(hw, I40E_PRTTSYN_STAT_0);
9436         if ((sync_status & I40E_PRTTSYN_STAT_0_TXTIME_MASK) == 0)
9437                 return -EINVAL;
9438
9439         tx_tstamp_cycles = i40e_read_tx_tstamp_cyclecounter(dev);
9440         ns = rte_timecounter_update(&adapter->tx_tstamp_tc, tx_tstamp_cycles);
9441         *timestamp = rte_ns_to_timespec(ns);
9442
9443         return 0;
9444 }
9445
9446 /*
9447  * i40e_parse_dcb_configure - parse dcb configure from user
9448  * @dev: the device being configured
9449  * @dcb_cfg: pointer of the result of parse
9450  * @*tc_map: bit map of enabled traffic classes
9451  *
9452  * Returns 0 on success, negative value on failure
9453  */
9454 static int
9455 i40e_parse_dcb_configure(struct rte_eth_dev *dev,
9456                          struct i40e_dcbx_config *dcb_cfg,
9457                          uint8_t *tc_map)
9458 {
9459         struct rte_eth_dcb_rx_conf *dcb_rx_conf;
9460         uint8_t i, tc_bw, bw_lf;
9461
9462         memset(dcb_cfg, 0, sizeof(struct i40e_dcbx_config));
9463
9464         dcb_rx_conf = &dev->data->dev_conf.rx_adv_conf.dcb_rx_conf;
9465         if (dcb_rx_conf->nb_tcs > I40E_MAX_TRAFFIC_CLASS) {
9466                 PMD_INIT_LOG(ERR, "number of tc exceeds max.");
9467                 return -EINVAL;
9468         }
9469
9470         /* assume each tc has the same bw */
9471         tc_bw = I40E_MAX_PERCENT / dcb_rx_conf->nb_tcs;
9472         for (i = 0; i < dcb_rx_conf->nb_tcs; i++)
9473                 dcb_cfg->etscfg.tcbwtable[i] = tc_bw;
9474         /* to ensure the sum of tcbw is equal to 100 */
9475         bw_lf = I40E_MAX_PERCENT % dcb_rx_conf->nb_tcs;
9476         for (i = 0; i < bw_lf; i++)
9477                 dcb_cfg->etscfg.tcbwtable[i]++;
9478
9479         /* assume each tc has the same Transmission Selection Algorithm */
9480         for (i = 0; i < dcb_rx_conf->nb_tcs; i++)
9481                 dcb_cfg->etscfg.tsatable[i] = I40E_IEEE_TSA_ETS;
9482
9483         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
9484                 dcb_cfg->etscfg.prioritytable[i] =
9485                                 dcb_rx_conf->dcb_tc[i];
9486
9487         /* FW needs one App to configure HW */
9488         dcb_cfg->numapps = I40E_DEFAULT_DCB_APP_NUM;
9489         dcb_cfg->app[0].selector = I40E_APP_SEL_ETHTYPE;
9490         dcb_cfg->app[0].priority = I40E_DEFAULT_DCB_APP_PRIO;
9491         dcb_cfg->app[0].protocolid = I40E_APP_PROTOID_FCOE;
9492
9493         if (dcb_rx_conf->nb_tcs == 0)
9494                 *tc_map = 1; /* tc0 only */
9495         else
9496                 *tc_map = RTE_LEN2MASK(dcb_rx_conf->nb_tcs, uint8_t);
9497
9498         if (dev->data->dev_conf.dcb_capability_en & ETH_DCB_PFC_SUPPORT) {
9499                 dcb_cfg->pfc.willing = 0;
9500                 dcb_cfg->pfc.pfccap = I40E_MAX_TRAFFIC_CLASS;
9501                 dcb_cfg->pfc.pfcenable = *tc_map;
9502         }
9503         return 0;
9504 }
9505
9506
9507 static enum i40e_status_code
9508 i40e_vsi_update_queue_mapping(struct i40e_vsi *vsi,
9509                               struct i40e_aqc_vsi_properties_data *info,
9510                               uint8_t enabled_tcmap)
9511 {
9512         enum i40e_status_code ret;
9513         int i, total_tc = 0;
9514         uint16_t qpnum_per_tc, bsf, qp_idx;
9515         struct rte_eth_dev_data *dev_data = I40E_VSI_TO_DEV_DATA(vsi);
9516         struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
9517         uint16_t used_queues;
9518
9519         ret = validate_tcmap_parameter(vsi, enabled_tcmap);
9520         if (ret != I40E_SUCCESS)
9521                 return ret;
9522
9523         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
9524                 if (enabled_tcmap & (1 << i))
9525                         total_tc++;
9526         }
9527         if (total_tc == 0)
9528                 total_tc = 1;
9529         vsi->enabled_tc = enabled_tcmap;
9530
9531         /* different VSI has different queues assigned */
9532         if (vsi->type == I40E_VSI_MAIN)
9533                 used_queues = dev_data->nb_rx_queues -
9534                         pf->nb_cfg_vmdq_vsi * RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM;
9535         else if (vsi->type == I40E_VSI_VMDQ2)
9536                 used_queues = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM;
9537         else {
9538                 PMD_INIT_LOG(ERR, "unsupported VSI type.");
9539                 return I40E_ERR_NO_AVAILABLE_VSI;
9540         }
9541
9542         qpnum_per_tc = used_queues / total_tc;
9543         /* Number of queues per enabled TC */
9544         if (qpnum_per_tc == 0) {
9545                 PMD_INIT_LOG(ERR, " number of queues is less that tcs.");
9546                 return I40E_ERR_INVALID_QP_ID;
9547         }
9548         qpnum_per_tc = RTE_MIN(i40e_align_floor(qpnum_per_tc),
9549                                 I40E_MAX_Q_PER_TC);
9550         bsf = rte_bsf32(qpnum_per_tc);
9551
9552         /**
9553          * Configure TC and queue mapping parameters, for enabled TC,
9554          * allocate qpnum_per_tc queues to this traffic. For disabled TC,
9555          * default queue will serve it.
9556          */
9557         qp_idx = 0;
9558         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
9559                 if (vsi->enabled_tc & (1 << i)) {
9560                         info->tc_mapping[i] = rte_cpu_to_le_16((qp_idx <<
9561                                         I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
9562                                 (bsf << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT));
9563                         qp_idx += qpnum_per_tc;
9564                 } else
9565                         info->tc_mapping[i] = 0;
9566         }
9567
9568         /* Associate queue number with VSI, Keep vsi->nb_qps unchanged */
9569         if (vsi->type == I40E_VSI_SRIOV) {
9570                 info->mapping_flags |=
9571                         rte_cpu_to_le_16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
9572                 for (i = 0; i < vsi->nb_qps; i++)
9573                         info->queue_mapping[i] =
9574                                 rte_cpu_to_le_16(vsi->base_queue + i);
9575         } else {
9576                 info->mapping_flags |=
9577                         rte_cpu_to_le_16(I40E_AQ_VSI_QUE_MAP_CONTIG);
9578                 info->queue_mapping[0] = rte_cpu_to_le_16(vsi->base_queue);
9579         }
9580         info->valid_sections |=
9581                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID);
9582
9583         return I40E_SUCCESS;
9584 }
9585
9586 /*
9587  * i40e_config_switch_comp_tc - Configure VEB tc setting for given TC map
9588  * @veb: VEB to be configured
9589  * @tc_map: enabled TC bitmap
9590  *
9591  * Returns 0 on success, negative value on failure
9592  */
9593 static enum i40e_status_code
9594 i40e_config_switch_comp_tc(struct i40e_veb *veb, uint8_t tc_map)
9595 {
9596         struct i40e_aqc_configure_switching_comp_bw_config_data veb_bw;
9597         struct i40e_aqc_query_switching_comp_bw_config_resp bw_query;
9598         struct i40e_aqc_query_switching_comp_ets_config_resp ets_query;
9599         struct i40e_hw *hw = I40E_VSI_TO_HW(veb->associate_vsi);
9600         enum i40e_status_code ret = I40E_SUCCESS;
9601         int i;
9602         uint32_t bw_max;
9603
9604         /* Check if enabled_tc is same as existing or new TCs */
9605         if (veb->enabled_tc == tc_map)
9606                 return ret;
9607
9608         /* configure tc bandwidth */
9609         memset(&veb_bw, 0, sizeof(veb_bw));
9610         veb_bw.tc_valid_bits = tc_map;
9611         /* Enable ETS TCs with equal BW Share for now across all VSIs */
9612         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
9613                 if (tc_map & BIT_ULL(i))
9614                         veb_bw.tc_bw_share_credits[i] = 1;
9615         }
9616         ret = i40e_aq_config_switch_comp_bw_config(hw, veb->seid,
9617                                                    &veb_bw, NULL);
9618         if (ret) {
9619                 PMD_INIT_LOG(ERR, "AQ command Config switch_comp BW allocation"
9620                                   " per TC failed = %d",
9621                                   hw->aq.asq_last_status);
9622                 return ret;
9623         }
9624
9625         memset(&ets_query, 0, sizeof(ets_query));
9626         ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,
9627                                                    &ets_query, NULL);
9628         if (ret != I40E_SUCCESS) {
9629                 PMD_DRV_LOG(ERR, "Failed to get switch_comp ETS"
9630                                  " configuration %u", hw->aq.asq_last_status);
9631                 return ret;
9632         }
9633         memset(&bw_query, 0, sizeof(bw_query));
9634         ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,
9635                                                   &bw_query, NULL);
9636         if (ret != I40E_SUCCESS) {
9637                 PMD_DRV_LOG(ERR, "Failed to get switch_comp bandwidth"
9638                                  " configuration %u", hw->aq.asq_last_status);
9639                 return ret;
9640         }
9641
9642         /* store and print out BW info */
9643         veb->bw_info.bw_limit = rte_le_to_cpu_16(ets_query.port_bw_limit);
9644         veb->bw_info.bw_max = ets_query.tc_bw_max;
9645         PMD_DRV_LOG(DEBUG, "switch_comp bw limit:%u", veb->bw_info.bw_limit);
9646         PMD_DRV_LOG(DEBUG, "switch_comp max_bw:%u", veb->bw_info.bw_max);
9647         bw_max = rte_le_to_cpu_16(bw_query.tc_bw_max[0]) |
9648                     (rte_le_to_cpu_16(bw_query.tc_bw_max[1]) <<
9649                      I40E_16_BIT_WIDTH);
9650         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
9651                 veb->bw_info.bw_ets_share_credits[i] =
9652                                 bw_query.tc_bw_share_credits[i];
9653                 veb->bw_info.bw_ets_credits[i] =
9654                                 rte_le_to_cpu_16(bw_query.tc_bw_limits[i]);
9655                 /* 4 bits per TC, 4th bit is reserved */
9656                 veb->bw_info.bw_ets_max[i] =
9657                         (uint8_t)((bw_max >> (i * I40E_4_BIT_WIDTH)) &
9658                                   RTE_LEN2MASK(3, uint8_t));
9659                 PMD_DRV_LOG(DEBUG, "\tVEB TC%u:share credits %u", i,
9660                             veb->bw_info.bw_ets_share_credits[i]);
9661                 PMD_DRV_LOG(DEBUG, "\tVEB TC%u:credits %u", i,
9662                             veb->bw_info.bw_ets_credits[i]);
9663                 PMD_DRV_LOG(DEBUG, "\tVEB TC%u: max credits: %u", i,
9664                             veb->bw_info.bw_ets_max[i]);
9665         }
9666
9667         veb->enabled_tc = tc_map;
9668
9669         return ret;
9670 }
9671
9672
9673 /*
9674  * i40e_vsi_config_tc - Configure VSI tc setting for given TC map
9675  * @vsi: VSI to be configured
9676  * @tc_map: enabled TC bitmap
9677  *
9678  * Returns 0 on success, negative value on failure
9679  */
9680 static enum i40e_status_code
9681 i40e_vsi_config_tc(struct i40e_vsi *vsi, uint8_t tc_map)
9682 {
9683         struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
9684         struct i40e_vsi_context ctxt;
9685         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
9686         enum i40e_status_code ret = I40E_SUCCESS;
9687         int i;
9688
9689         /* Check if enabled_tc is same as existing or new TCs */
9690         if (vsi->enabled_tc == tc_map)
9691                 return ret;
9692
9693         /* configure tc bandwidth */
9694         memset(&bw_data, 0, sizeof(bw_data));
9695         bw_data.tc_valid_bits = tc_map;
9696         /* Enable ETS TCs with equal BW Share for now across all VSIs */
9697         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
9698                 if (tc_map & BIT_ULL(i))
9699                         bw_data.tc_bw_credits[i] = 1;
9700         }
9701         ret = i40e_aq_config_vsi_tc_bw(hw, vsi->seid, &bw_data, NULL);
9702         if (ret) {
9703                 PMD_INIT_LOG(ERR, "AQ command Config VSI BW allocation"
9704                         " per TC failed = %d",
9705                         hw->aq.asq_last_status);
9706                 goto out;
9707         }
9708         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
9709                 vsi->info.qs_handle[i] = bw_data.qs_handles[i];
9710
9711         /* Update Queue Pairs Mapping for currently enabled UPs */
9712         ctxt.seid = vsi->seid;
9713         ctxt.pf_num = hw->pf_id;
9714         ctxt.vf_num = 0;
9715         ctxt.uplink_seid = vsi->uplink_seid;
9716         ctxt.info = vsi->info;
9717         i40e_get_cap(hw);
9718         ret = i40e_vsi_update_queue_mapping(vsi, &ctxt.info, tc_map);
9719         if (ret)
9720                 goto out;
9721
9722         /* Update the VSI after updating the VSI queue-mapping information */
9723         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
9724         if (ret) {
9725                 PMD_INIT_LOG(ERR, "Failed to configure "
9726                             "TC queue mapping = %d",
9727                             hw->aq.asq_last_status);
9728                 goto out;
9729         }
9730         /* update the local VSI info with updated queue map */
9731         (void)rte_memcpy(&vsi->info.tc_mapping, &ctxt.info.tc_mapping,
9732                                         sizeof(vsi->info.tc_mapping));
9733         (void)rte_memcpy(&vsi->info.queue_mapping,
9734                         &ctxt.info.queue_mapping,
9735                 sizeof(vsi->info.queue_mapping));
9736         vsi->info.mapping_flags = ctxt.info.mapping_flags;
9737         vsi->info.valid_sections = 0;
9738
9739         /* query and update current VSI BW information */
9740         ret = i40e_vsi_get_bw_config(vsi);
9741         if (ret) {
9742                 PMD_INIT_LOG(ERR,
9743                          "Failed updating vsi bw info, err %s aq_err %s",
9744                          i40e_stat_str(hw, ret),
9745                          i40e_aq_str(hw, hw->aq.asq_last_status));
9746                 goto out;
9747         }
9748
9749         vsi->enabled_tc = tc_map;
9750
9751 out:
9752         return ret;
9753 }
9754
9755 /*
9756  * i40e_dcb_hw_configure - program the dcb setting to hw
9757  * @pf: pf the configuration is taken on
9758  * @new_cfg: new configuration
9759  * @tc_map: enabled TC bitmap
9760  *
9761  * Returns 0 on success, negative value on failure
9762  */
9763 static enum i40e_status_code
9764 i40e_dcb_hw_configure(struct i40e_pf *pf,
9765                       struct i40e_dcbx_config *new_cfg,
9766                       uint8_t tc_map)
9767 {
9768         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
9769         struct i40e_dcbx_config *old_cfg = &hw->local_dcbx_config;
9770         struct i40e_vsi *main_vsi = pf->main_vsi;
9771         struct i40e_vsi_list *vsi_list;
9772         enum i40e_status_code ret;
9773         int i;
9774         uint32_t val;
9775
9776         /* Use the FW API if FW > v4.4*/
9777         if (!(((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver >= 4)) ||
9778               (hw->aq.fw_maj_ver >= 5))) {
9779                 PMD_INIT_LOG(ERR, "FW < v4.4, can not use FW LLDP API"
9780                                   " to configure DCB");
9781                 return I40E_ERR_FIRMWARE_API_VERSION;
9782         }
9783
9784         /* Check if need reconfiguration */
9785         if (!memcmp(new_cfg, old_cfg, sizeof(struct i40e_dcbx_config))) {
9786                 PMD_INIT_LOG(ERR, "No Change in DCB Config required.");
9787                 return I40E_SUCCESS;
9788         }
9789
9790         /* Copy the new config to the current config */
9791         *old_cfg = *new_cfg;
9792         old_cfg->etsrec = old_cfg->etscfg;
9793         ret = i40e_set_dcb_config(hw);
9794         if (ret) {
9795                 PMD_INIT_LOG(ERR,
9796                          "Set DCB Config failed, err %s aq_err %s\n",
9797                          i40e_stat_str(hw, ret),
9798                          i40e_aq_str(hw, hw->aq.asq_last_status));
9799                 return ret;
9800         }
9801         /* set receive Arbiter to RR mode and ETS scheme by default */
9802         for (i = 0; i <= I40E_PRTDCB_RETSTCC_MAX_INDEX; i++) {
9803                 val = I40E_READ_REG(hw, I40E_PRTDCB_RETSTCC(i));
9804                 val &= ~(I40E_PRTDCB_RETSTCC_BWSHARE_MASK     |
9805                          I40E_PRTDCB_RETSTCC_UPINTC_MODE_MASK |
9806                          I40E_PRTDCB_RETSTCC_ETSTC_SHIFT);
9807                 val |= ((uint32_t)old_cfg->etscfg.tcbwtable[i] <<
9808                         I40E_PRTDCB_RETSTCC_BWSHARE_SHIFT) &
9809                          I40E_PRTDCB_RETSTCC_BWSHARE_MASK;
9810                 val |= ((uint32_t)1 << I40E_PRTDCB_RETSTCC_UPINTC_MODE_SHIFT) &
9811                          I40E_PRTDCB_RETSTCC_UPINTC_MODE_MASK;
9812                 val |= ((uint32_t)1 << I40E_PRTDCB_RETSTCC_ETSTC_SHIFT) &
9813                          I40E_PRTDCB_RETSTCC_ETSTC_MASK;
9814                 I40E_WRITE_REG(hw, I40E_PRTDCB_RETSTCC(i), val);
9815         }
9816         /* get local mib to check whether it is configured correctly */
9817         /* IEEE mode */
9818         hw->local_dcbx_config.dcbx_mode = I40E_DCBX_MODE_IEEE;
9819         /* Get Local DCB Config */
9820         i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_LOCAL, 0,
9821                                      &hw->local_dcbx_config);
9822
9823         /* if Veb is created, need to update TC of it at first */
9824         if (main_vsi->veb) {
9825                 ret = i40e_config_switch_comp_tc(main_vsi->veb, tc_map);
9826                 if (ret)
9827                         PMD_INIT_LOG(WARNING,
9828                                  "Failed configuring TC for VEB seid=%d\n",
9829                                  main_vsi->veb->seid);
9830         }
9831         /* Update each VSI */
9832         i40e_vsi_config_tc(main_vsi, tc_map);
9833         if (main_vsi->veb) {
9834                 TAILQ_FOREACH(vsi_list, &main_vsi->veb->head, list) {
9835                         /* Beside main VSI and VMDQ VSIs, only enable default
9836                          * TC for other VSIs
9837                          */
9838                         if (vsi_list->vsi->type == I40E_VSI_VMDQ2)
9839                                 ret = i40e_vsi_config_tc(vsi_list->vsi,
9840                                                          tc_map);
9841                         else
9842                                 ret = i40e_vsi_config_tc(vsi_list->vsi,
9843                                                          I40E_DEFAULT_TCMAP);
9844                         if (ret)
9845                                 PMD_INIT_LOG(WARNING,
9846                                          "Failed configuring TC for VSI seid=%d\n",
9847                                          vsi_list->vsi->seid);
9848                         /* continue */
9849                 }
9850         }
9851         return I40E_SUCCESS;
9852 }
9853
9854 /*
9855  * i40e_dcb_init_configure - initial dcb config
9856  * @dev: device being configured
9857  * @sw_dcb: indicate whether dcb is sw configured or hw offload
9858  *
9859  * Returns 0 on success, negative value on failure
9860  */
9861 static int
9862 i40e_dcb_init_configure(struct rte_eth_dev *dev, bool sw_dcb)
9863 {
9864         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
9865         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
9866         int ret = 0;
9867
9868         if ((pf->flags & I40E_FLAG_DCB) == 0) {
9869                 PMD_INIT_LOG(ERR, "HW doesn't support DCB");
9870                 return -ENOTSUP;
9871         }
9872
9873         /* DCB initialization:
9874          * Update DCB configuration from the Firmware and configure
9875          * LLDP MIB change event.
9876          */
9877         if (sw_dcb == TRUE) {
9878                 ret = i40e_init_dcb(hw);
9879                 /* If lldp agent is stopped, the return value from
9880                  * i40e_init_dcb we expect is failure with I40E_AQ_RC_EPERM
9881                  * adminq status. Otherwise, it should return success.
9882                  */
9883                 if ((ret == I40E_SUCCESS) || (ret != I40E_SUCCESS &&
9884                     hw->aq.asq_last_status == I40E_AQ_RC_EPERM)) {
9885                         memset(&hw->local_dcbx_config, 0,
9886                                 sizeof(struct i40e_dcbx_config));
9887                         /* set dcb default configuration */
9888                         hw->local_dcbx_config.etscfg.willing = 0;
9889                         hw->local_dcbx_config.etscfg.maxtcs = 0;
9890                         hw->local_dcbx_config.etscfg.tcbwtable[0] = 100;
9891                         hw->local_dcbx_config.etscfg.tsatable[0] =
9892                                                 I40E_IEEE_TSA_ETS;
9893                         hw->local_dcbx_config.etsrec =
9894                                 hw->local_dcbx_config.etscfg;
9895                         hw->local_dcbx_config.pfc.willing = 0;
9896                         hw->local_dcbx_config.pfc.pfccap =
9897                                                 I40E_MAX_TRAFFIC_CLASS;
9898                         /* FW needs one App to configure HW */
9899                         hw->local_dcbx_config.numapps = 1;
9900                         hw->local_dcbx_config.app[0].selector =
9901                                                 I40E_APP_SEL_ETHTYPE;
9902                         hw->local_dcbx_config.app[0].priority = 3;
9903                         hw->local_dcbx_config.app[0].protocolid =
9904                                                 I40E_APP_PROTOID_FCOE;
9905                         ret = i40e_set_dcb_config(hw);
9906                         if (ret) {
9907                                 PMD_INIT_LOG(ERR, "default dcb config fails."
9908                                         " err = %d, aq_err = %d.", ret,
9909                                           hw->aq.asq_last_status);
9910                                 return -ENOSYS;
9911                         }
9912                 } else {
9913                         PMD_INIT_LOG(ERR, "DCB initialization in FW fails,"
9914                                           " err = %d, aq_err = %d.", ret,
9915                                           hw->aq.asq_last_status);
9916                         return -ENOTSUP;
9917                 }
9918         } else {
9919                 ret = i40e_aq_start_lldp(hw, NULL);
9920                 if (ret != I40E_SUCCESS)
9921                         PMD_INIT_LOG(DEBUG, "Failed to start lldp");
9922
9923                 ret = i40e_init_dcb(hw);
9924                 if (!ret) {
9925                         if (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED) {
9926                                 PMD_INIT_LOG(ERR, "HW doesn't support"
9927                                                   " DCBX offload.");
9928                                 return -ENOTSUP;
9929                         }
9930                 } else {
9931                         PMD_INIT_LOG(ERR, "DCBX configuration failed, err = %d,"
9932                                           " aq_err = %d.", ret,
9933                                           hw->aq.asq_last_status);
9934                         return -ENOTSUP;
9935                 }
9936         }
9937         return 0;
9938 }
9939
9940 /*
9941  * i40e_dcb_setup - setup dcb related config
9942  * @dev: device being configured
9943  *
9944  * Returns 0 on success, negative value on failure
9945  */
9946 static int
9947 i40e_dcb_setup(struct rte_eth_dev *dev)
9948 {
9949         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
9950         struct i40e_dcbx_config dcb_cfg;
9951         uint8_t tc_map = 0;
9952         int ret = 0;
9953
9954         if ((pf->flags & I40E_FLAG_DCB) == 0) {
9955                 PMD_INIT_LOG(ERR, "HW doesn't support DCB");
9956                 return -ENOTSUP;
9957         }
9958
9959         if (pf->vf_num != 0)
9960                 PMD_INIT_LOG(DEBUG, " DCB only works on pf and vmdq vsis.");
9961
9962         ret = i40e_parse_dcb_configure(dev, &dcb_cfg, &tc_map);
9963         if (ret) {
9964                 PMD_INIT_LOG(ERR, "invalid dcb config");
9965                 return -EINVAL;
9966         }
9967         ret = i40e_dcb_hw_configure(pf, &dcb_cfg, tc_map);
9968         if (ret) {
9969                 PMD_INIT_LOG(ERR, "dcb sw configure fails");
9970                 return -ENOSYS;
9971         }
9972
9973         return 0;
9974 }
9975
9976 static int
9977 i40e_dev_get_dcb_info(struct rte_eth_dev *dev,
9978                       struct rte_eth_dcb_info *dcb_info)
9979 {
9980         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
9981         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
9982         struct i40e_vsi *vsi = pf->main_vsi;
9983         struct i40e_dcbx_config *dcb_cfg = &hw->local_dcbx_config;
9984         uint16_t bsf, tc_mapping;
9985         int i, j = 0;
9986
9987         if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_DCB_FLAG)
9988                 dcb_info->nb_tcs = rte_bsf32(vsi->enabled_tc + 1);
9989         else
9990                 dcb_info->nb_tcs = 1;
9991         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
9992                 dcb_info->prio_tc[i] = dcb_cfg->etscfg.prioritytable[i];
9993         for (i = 0; i < dcb_info->nb_tcs; i++)
9994                 dcb_info->tc_bws[i] = dcb_cfg->etscfg.tcbwtable[i];
9995
9996         /* get queue mapping if vmdq is disabled */
9997         if (!pf->nb_cfg_vmdq_vsi) {
9998                 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
9999                         if (!(vsi->enabled_tc & (1 << i)))
10000                                 continue;
10001                         tc_mapping = rte_le_to_cpu_16(vsi->info.tc_mapping[i]);
10002                         dcb_info->tc_queue.tc_rxq[j][i].base =
10003                                 (tc_mapping & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) >>
10004                                 I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT;
10005                         dcb_info->tc_queue.tc_txq[j][i].base =
10006                                 dcb_info->tc_queue.tc_rxq[j][i].base;
10007                         bsf = (tc_mapping & I40E_AQ_VSI_TC_QUE_NUMBER_MASK) >>
10008                                 I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT;
10009                         dcb_info->tc_queue.tc_rxq[j][i].nb_queue = 1 << bsf;
10010                         dcb_info->tc_queue.tc_txq[j][i].nb_queue =
10011                                 dcb_info->tc_queue.tc_rxq[j][i].nb_queue;
10012                 }
10013                 return 0;
10014         }
10015
10016         /* get queue mapping if vmdq is enabled */
10017         do {
10018                 vsi = pf->vmdq[j].vsi;
10019                 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
10020                         if (!(vsi->enabled_tc & (1 << i)))
10021                                 continue;
10022                         tc_mapping = rte_le_to_cpu_16(vsi->info.tc_mapping[i]);
10023                         dcb_info->tc_queue.tc_rxq[j][i].base =
10024                                 (tc_mapping & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) >>
10025                                 I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT;
10026                         dcb_info->tc_queue.tc_txq[j][i].base =
10027                                 dcb_info->tc_queue.tc_rxq[j][i].base;
10028                         bsf = (tc_mapping & I40E_AQ_VSI_TC_QUE_NUMBER_MASK) >>
10029                                 I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT;
10030                         dcb_info->tc_queue.tc_rxq[j][i].nb_queue = 1 << bsf;
10031                         dcb_info->tc_queue.tc_txq[j][i].nb_queue =
10032                                 dcb_info->tc_queue.tc_rxq[j][i].nb_queue;
10033                 }
10034                 j++;
10035         } while (j < RTE_MIN(pf->nb_cfg_vmdq_vsi, ETH_MAX_VMDQ_POOL));
10036         return 0;
10037 }
10038
10039 static int
10040 i40e_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
10041 {
10042         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
10043         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
10044         uint16_t msix_intr;
10045
10046         msix_intr = intr_handle->intr_vec[queue_id];
10047         if (msix_intr == I40E_MISC_VEC_ID)
10048                 I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0,
10049                                I40E_PFINT_DYN_CTL0_INTENA_MASK |
10050                                I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
10051                                I40E_PFINT_DYN_CTL0_ITR_INDX_MASK);
10052         else
10053                 I40E_WRITE_REG(hw,
10054                                I40E_PFINT_DYN_CTLN(msix_intr -
10055                                                    I40E_RX_VEC_START),
10056                                I40E_PFINT_DYN_CTLN_INTENA_MASK |
10057                                I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
10058                                I40E_PFINT_DYN_CTLN_ITR_INDX_MASK);
10059
10060         I40E_WRITE_FLUSH(hw);
10061         rte_intr_enable(&dev->pci_dev->intr_handle);
10062
10063         return 0;
10064 }
10065
10066 static int
10067 i40e_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
10068 {
10069         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
10070         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
10071         uint16_t msix_intr;
10072
10073         msix_intr = intr_handle->intr_vec[queue_id];
10074         if (msix_intr == I40E_MISC_VEC_ID)
10075                 I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0,
10076                                I40E_PFINT_DYN_CTL0_ITR_INDX_MASK);
10077         else
10078                 I40E_WRITE_REG(hw,
10079                                I40E_PFINT_DYN_CTLN(msix_intr -
10080                                                    I40E_RX_VEC_START),
10081                                I40E_PFINT_DYN_CTLN_ITR_INDX_MASK);
10082         I40E_WRITE_FLUSH(hw);
10083
10084         return 0;
10085 }
10086
10087 /**
10088  * This function is used to check if the register is valid.
10089  * Below is the valid registers list for X722 only:
10090  * 0x2b800--0x2bb00
10091  * 0x38700--0x38a00
10092  * 0x3d800--0x3db00
10093  * 0x208e00--0x209000
10094  * 0x20be00--0x20c000
10095  * 0x263c00--0x264000
10096  * 0x265c00--0x266000
10097  */
10098 static inline int i40e_valid_regs(enum i40e_mac_type type, uint32_t reg_offset)
10099 {
10100         if ((type != I40E_MAC_X722) &&
10101             ((reg_offset >= 0x2b800 && reg_offset <= 0x2bb00) ||
10102              (reg_offset >= 0x38700 && reg_offset <= 0x38a00) ||
10103              (reg_offset >= 0x3d800 && reg_offset <= 0x3db00) ||
10104              (reg_offset >= 0x208e00 && reg_offset <= 0x209000) ||
10105              (reg_offset >= 0x20be00 && reg_offset <= 0x20c000) ||
10106              (reg_offset >= 0x263c00 && reg_offset <= 0x264000) ||
10107              (reg_offset >= 0x265c00 && reg_offset <= 0x266000)))
10108                 return 0;
10109         else
10110                 return 1;
10111 }
10112
10113 static int i40e_get_regs(struct rte_eth_dev *dev,
10114                          struct rte_dev_reg_info *regs)
10115 {
10116         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
10117         uint32_t *ptr_data = regs->data;
10118         uint32_t reg_idx, arr_idx, arr_idx2, reg_offset;
10119         const struct i40e_reg_info *reg_info;
10120
10121         if (ptr_data == NULL) {
10122                 regs->length = I40E_GLGEN_STAT_CLEAR + 4;
10123                 regs->width = sizeof(uint32_t);
10124                 return 0;
10125         }
10126
10127         /* The first few registers have to be read using AQ operations */
10128         reg_idx = 0;
10129         while (i40e_regs_adminq[reg_idx].name) {
10130                 reg_info = &i40e_regs_adminq[reg_idx++];
10131                 for (arr_idx = 0; arr_idx <= reg_info->count1; arr_idx++)
10132                         for (arr_idx2 = 0;
10133                                         arr_idx2 <= reg_info->count2;
10134                                         arr_idx2++) {
10135                                 reg_offset = arr_idx * reg_info->stride1 +
10136                                         arr_idx2 * reg_info->stride2;
10137                                 reg_offset += reg_info->base_addr;
10138                                 ptr_data[reg_offset >> 2] =
10139                                         i40e_read_rx_ctl(hw, reg_offset);
10140                         }
10141         }
10142
10143         /* The remaining registers can be read using primitives */
10144         reg_idx = 0;
10145         while (i40e_regs_others[reg_idx].name) {
10146                 reg_info = &i40e_regs_others[reg_idx++];
10147                 for (arr_idx = 0; arr_idx <= reg_info->count1; arr_idx++)
10148                         for (arr_idx2 = 0;
10149                                         arr_idx2 <= reg_info->count2;
10150                                         arr_idx2++) {
10151                                 reg_offset = arr_idx * reg_info->stride1 +
10152                                         arr_idx2 * reg_info->stride2;
10153                                 reg_offset += reg_info->base_addr;
10154                                 if (!i40e_valid_regs(hw->mac.type, reg_offset))
10155                                         ptr_data[reg_offset >> 2] = 0;
10156                                 else
10157                                         ptr_data[reg_offset >> 2] =
10158                                                 I40E_READ_REG(hw, reg_offset);
10159                         }
10160         }
10161
10162         return 0;
10163 }
10164
10165 static int i40e_get_eeprom_length(struct rte_eth_dev *dev)
10166 {
10167         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
10168
10169         /* Convert word count to byte count */
10170         return hw->nvm.sr_size << 1;
10171 }
10172
10173 static int i40e_get_eeprom(struct rte_eth_dev *dev,
10174                            struct rte_dev_eeprom_info *eeprom)
10175 {
10176         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
10177         uint16_t *data = eeprom->data;
10178         uint16_t offset, length, cnt_words;
10179         int ret_code;
10180
10181         offset = eeprom->offset >> 1;
10182         length = eeprom->length >> 1;
10183         cnt_words = length;
10184
10185         if (offset > hw->nvm.sr_size ||
10186                 offset + length > hw->nvm.sr_size) {
10187                 PMD_DRV_LOG(ERR, "Requested EEPROM bytes out of range.");
10188                 return -EINVAL;
10189         }
10190
10191         eeprom->magic = hw->vendor_id | (hw->device_id << 16);
10192
10193         ret_code = i40e_read_nvm_buffer(hw, offset, &cnt_words, data);
10194         if (ret_code != I40E_SUCCESS || cnt_words != length) {
10195                 PMD_DRV_LOG(ERR, "EEPROM read failed.");
10196                 return -EIO;
10197         }
10198
10199         return 0;
10200 }
10201
10202 static void i40e_set_default_mac_addr(struct rte_eth_dev *dev,
10203                                       struct ether_addr *mac_addr)
10204 {
10205         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
10206         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
10207         struct i40e_vsi *vsi = pf->main_vsi;
10208         struct i40e_mac_filter_info mac_filter;
10209         struct i40e_mac_filter *f;
10210         int ret;
10211
10212         if (!is_valid_assigned_ether_addr(mac_addr)) {
10213                 PMD_DRV_LOG(ERR, "Tried to set invalid MAC address.");
10214                 return;
10215         }
10216
10217         TAILQ_FOREACH(f, &vsi->mac_list, next) {
10218                 if (is_same_ether_addr(&pf->dev_addr, &f->mac_info.mac_addr))
10219                         break;
10220         }
10221
10222         if (f == NULL) {
10223                 PMD_DRV_LOG(ERR, "Failed to find filter for default mac");
10224                 return;
10225         }
10226
10227         mac_filter = f->mac_info;
10228         ret = i40e_vsi_delete_mac(vsi, &mac_filter.mac_addr);
10229         if (ret != I40E_SUCCESS) {
10230                 PMD_DRV_LOG(ERR, "Failed to delete mac filter");
10231                 return;
10232         }
10233         memcpy(&mac_filter.mac_addr, mac_addr, ETH_ADDR_LEN);
10234         ret = i40e_vsi_add_mac(vsi, &mac_filter);
10235         if (ret != I40E_SUCCESS) {
10236                 PMD_DRV_LOG(ERR, "Failed to add mac filter");
10237                 return;
10238         }
10239         memcpy(&pf->dev_addr, mac_addr, ETH_ADDR_LEN);
10240
10241         i40e_aq_mac_address_write(hw, I40E_AQC_WRITE_TYPE_LAA_WOL,
10242                                   mac_addr->addr_bytes, NULL);
10243 }
10244
10245 static int
10246 i40e_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
10247 {
10248         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
10249         struct rte_eth_dev_data *dev_data = pf->dev_data;
10250         uint32_t frame_size = mtu + ETHER_HDR_LEN
10251                               + ETHER_CRC_LEN + I40E_VLAN_TAG_SIZE;
10252         int ret = 0;
10253
10254         /* check if mtu is within the allowed range */
10255         if ((mtu < ETHER_MIN_MTU) || (frame_size > I40E_FRAME_SIZE_MAX))
10256                 return -EINVAL;
10257
10258         /* mtu setting is forbidden if port is start */
10259         if (dev_data->dev_started) {
10260                 PMD_DRV_LOG(ERR,
10261                             "port %d must be stopped before configuration\n",
10262                             dev_data->port_id);
10263                 return -EBUSY;
10264         }
10265
10266         if (frame_size > ETHER_MAX_LEN)
10267                 dev_data->dev_conf.rxmode.jumbo_frame = 1;
10268         else
10269                 dev_data->dev_conf.rxmode.jumbo_frame = 0;
10270
10271         dev_data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
10272
10273         return ret;
10274 }