drivers/net/fm10k/fm10k_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2013-2016 Intel Corporation
   3  */
   4
   5 #include <rte_ethdev_driver.h>
   6 #include <rte_ethdev_pci.h>
   7 #include <rte_malloc.h>
   8 #include <rte_memzone.h>
   9 #include <rte_string_fns.h>
  10 #include <rte_dev.h>
  11 #include <rte_spinlock.h>
  12 #include <rte_kvargs.h>
  13
  14 #include "fm10k.h"
  15 #include "base/fm10k_api.h"
  16
  17 /* Default delay to acquire mailbox lock */
  18 #define FM10K_MBXLOCK_DELAY_US 20
  19 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  20
  21 #define MAIN_VSI_POOL_NUMBER 0
  22
  23 /* Max try times to acquire switch status */
  24 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  25 /* Wait interval to get switch status */
  26 #define WAIT_SWITCH_MSG_US    100000
  27 /* A period of quiescence for switch */
  28 #define FM10K_SWITCH_QUIESCE_US 100000
  29 /* Number of chars per uint32 type */
  30 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  31 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  32
  33 /* default 1:1 map from queue ID to interrupt vector ID */
  34 #define Q2V(pci_dev, queue_id) ((pci_dev)->intr_handle.intr_vec[queue_id])
  35
  36 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  37 #define MAX_LPORT_NUM    128
  38 #define GLORT_FD_Q_BASE  0x40
  39 #define GLORT_PF_MASK    0xFFC0
  40 #define GLORT_FD_MASK    GLORT_PF_MASK
  41 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  42
  43 int fm10k_logtype_init;
  44 int fm10k_logtype_driver;
  45
  46 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  47 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  48 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  49 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  50 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  51 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  52 static int
  53 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  54 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  55         const u8 *mac, bool add, uint32_t pool);
  56 static void fm10k_tx_queue_release(void *queue);
  57 static void fm10k_rx_queue_release(void *queue);
  58 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  59 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  60 static int fm10k_check_ftag(struct rte_devargs *devargs);
  61 static int fm10k_link_update(struct rte_eth_dev *dev, int wait_to_complete);
  62
  63 static void fm10k_dev_infos_get(struct rte_eth_dev *dev,
  64                                 struct rte_eth_dev_info *dev_info);
  65 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev);
  66 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev);
  67 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev);
  68 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev);
  69
  70 struct fm10k_xstats_name_off {
  71         char name[RTE_ETH_XSTATS_NAME_SIZE];
  72         unsigned offset;
  73 };
  74
  75 static const struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  76         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  77         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  78         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  79         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  80         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  81         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
  82         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
  83         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
  84                 nodesc_drop)},
  85 };
  86
  87 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
  88                 sizeof(fm10k_hw_stats_strings[0]))
  89
  90 static const struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
  91         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
  92         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
  93         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
  94 };
  95
  96 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
  97                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
  98
  99 static const struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
 100         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 101         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 102 };
 103
 104 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 105                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 106
 107 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 108                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 109 static int
 110 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 111
 112 static void
 113 fm10k_mbx_initlock(struct fm10k_hw *hw)
 114 {
 115         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 116 }
 117
 118 static void
 119 fm10k_mbx_lock(struct fm10k_hw *hw)
 120 {
 121         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 122                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 123 }
 124
 125 static void
 126 fm10k_mbx_unlock(struct fm10k_hw *hw)
 127 {
 128         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 129 }
 130
 131 /* Stubs needed for linkage when vPMD is disabled */
 132 __rte_weak int
 133 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 134 {
 135         return -1;
 136 }
 137
 138 __rte_weak uint16_t
 139 fm10k_recv_pkts_vec(
 140         __rte_unused void *rx_queue,
 141         __rte_unused struct rte_mbuf **rx_pkts,
 142         __rte_unused uint16_t nb_pkts)
 143 {
 144         return 0;
 145 }
 146
 147 __rte_weak uint16_t
 148 fm10k_recv_scattered_pkts_vec(
 149                 __rte_unused void *rx_queue,
 150                 __rte_unused struct rte_mbuf **rx_pkts,
 151                 __rte_unused uint16_t nb_pkts)
 152 {
 153         return 0;
 154 }
 155
 156 __rte_weak int
 157 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 158
 159 {
 160         return -1;
 161 }
 162
 163 __rte_weak void
 164 fm10k_rx_queue_release_mbufs_vec(
 165                 __rte_unused struct fm10k_rx_queue *rxq)
 166 {
 167         return;
 168 }
 169
 170 __rte_weak void
 171 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 172 {
 173         return;
 174 }
 175
 176 __rte_weak int
 177 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 178 {
 179         return -1;
 180 }
 181
 182 __rte_weak uint16_t
 183 fm10k_xmit_fixed_burst_vec(__rte_unused void *tx_queue,
 184                            __rte_unused struct rte_mbuf **tx_pkts,
 185                            __rte_unused uint16_t nb_pkts)
 186 {
 187         return 0;
 188 }
 189
 190 /*
 191  * reset queue to initial state, allocate software buffers used when starting
 192  * device.
 193  * return 0 on success
 194  * return -ENOMEM if buffers cannot be allocated
 195  * return -EINVAL if buffers do not satisfy alignment condition
 196  */
 197 static inline int
 198 rx_queue_reset(struct fm10k_rx_queue *q)
 199 {
 200         static const union fm10k_rx_desc zero = {{0} };
 201         uint64_t dma_addr;
 202         int i, diag;
 203         PMD_INIT_FUNC_TRACE();
 204
 205         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 206         if (diag != 0)
 207                 return -ENOMEM;
 208
 209         for (i = 0; i < q->nb_desc; ++i) {
 210                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 211                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 212                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 213                                                 q->nb_desc);
 214                         return -EINVAL;
 215                 }
 216                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 217                 q->hw_ring[i].q.pkt_addr = dma_addr;
 218                 q->hw_ring[i].q.hdr_addr = dma_addr;
 219         }
 220
 221         /* initialize extra software ring entries. Space for these extra
 222          * entries is always allocated.
 223          */
 224         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 225         for (i = 0; i < q->nb_fake_desc; ++i) {
 226                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 227                 q->hw_ring[q->nb_desc + i] = zero;
 228         }
 229
 230         q->next_dd = 0;
 231         q->next_alloc = 0;
 232         q->next_trigger = q->alloc_thresh - 1;
 233         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 234         q->rxrearm_start = 0;
 235         q->rxrearm_nb = 0;
 236
 237         return 0;
 238 }
 239
 240 /*
 241  * clean queue, descriptor rings, free software buffers used when stopping
 242  * device.
 243  */
 244 static inline void
 245 rx_queue_clean(struct fm10k_rx_queue *q)
 246 {
 247         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 248         uint32_t i;
 249         PMD_INIT_FUNC_TRACE();
 250
 251         /* zero descriptor rings */
 252         for (i = 0; i < q->nb_desc; ++i)
 253                 q->hw_ring[i] = zero;
 254
 255         /* zero faked descriptors */
 256         for (i = 0; i < q->nb_fake_desc; ++i)
 257                 q->hw_ring[q->nb_desc + i] = zero;
 258
 259         /* vPMD driver has a different way of releasing mbufs. */
 260         if (q->rx_using_sse) {
 261                 fm10k_rx_queue_release_mbufs_vec(q);
 262                 return;
 263         }
 264
 265         /* free software buffers */
 266         for (i = 0; i < q->nb_desc; ++i) {
 267                 if (q->sw_ring[i]) {
 268                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 269                         q->sw_ring[i] = NULL;
 270                 }
 271         }
 272 }
 273
 274 /*
 275  * free all queue memory used when releasing the queue (i.e. configure)
 276  */
 277 static inline void
 278 rx_queue_free(struct fm10k_rx_queue *q)
 279 {
 280         PMD_INIT_FUNC_TRACE();
 281         if (q) {
 282                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 283                 rx_queue_clean(q);
 284                 if (q->sw_ring) {
 285                         rte_free(q->sw_ring);
 286                         q->sw_ring = NULL;
 287                 }
 288                 rte_free(q);
 289                 q = NULL;
 290         }
 291 }
 292
 293 /*
 294  * disable RX queue, wait unitl HW finished necessary flush operation
 295  */
 296 static inline int
 297 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 298 {
 299         uint32_t reg, i;
 300
 301         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 302         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 303                         reg & ~FM10K_RXQCTL_ENABLE);
 304
 305         /* Wait 100us at most */
 306         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 307                 rte_delay_us(1);
 308                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 309                 if (!(reg & FM10K_RXQCTL_ENABLE))
 310                         break;
 311         }
 312
 313         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 314                 return -1;
 315
 316         return 0;
 317 }
 318
 319 /*
 320  * reset queue to initial state, allocate software buffers used when starting
 321  * device
 322  */
 323 static inline void
 324 tx_queue_reset(struct fm10k_tx_queue *q)
 325 {
 326         PMD_INIT_FUNC_TRACE();
 327         q->last_free = 0;
 328         q->next_free = 0;
 329         q->nb_used = 0;
 330         q->nb_free = q->nb_desc - 1;
 331         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 332         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 333 }
 334
 335 /*
 336  * clean queue, descriptor rings, free software buffers used when stopping
 337  * device
 338  */
 339 static inline void
 340 tx_queue_clean(struct fm10k_tx_queue *q)
 341 {
 342         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 343         uint32_t i;
 344         PMD_INIT_FUNC_TRACE();
 345
 346         /* zero descriptor rings */
 347         for (i = 0; i < q->nb_desc; ++i)
 348                 q->hw_ring[i] = zero;
 349
 350         /* free software buffers */
 351         for (i = 0; i < q->nb_desc; ++i) {
 352                 if (q->sw_ring[i]) {
 353                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 354                         q->sw_ring[i] = NULL;
 355                 }
 356         }
 357 }
 358
 359 /*
 360  * free all queue memory used when releasing the queue (i.e. configure)
 361  */
 362 static inline void
 363 tx_queue_free(struct fm10k_tx_queue *q)
 364 {
 365         PMD_INIT_FUNC_TRACE();
 366         if (q) {
 367                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 368                 tx_queue_clean(q);
 369                 if (q->rs_tracker.list) {
 370                         rte_free(q->rs_tracker.list);
 371                         q->rs_tracker.list = NULL;
 372                 }
 373                 if (q->sw_ring) {
 374                         rte_free(q->sw_ring);
 375                         q->sw_ring = NULL;
 376                 }
 377                 rte_free(q);
 378                 q = NULL;
 379         }
 380 }
 381
 382 /*
 383  * disable TX queue, wait unitl HW finished necessary flush operation
 384  */
 385 static inline int
 386 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 387 {
 388         uint32_t reg, i;
 389
 390         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 391         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 392                         reg & ~FM10K_TXDCTL_ENABLE);
 393
 394         /* Wait 100us at most */
 395         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 396                 rte_delay_us(1);
 397                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 398                 if (!(reg & FM10K_TXDCTL_ENABLE))
 399                         break;
 400         }
 401
 402         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 403                 return -1;
 404
 405         return 0;
 406 }
 407
 408 static int
 409 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 410 {
 411         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 412         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 413         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 414         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 415
 416         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 417
 418         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 419                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 420                 return -EINVAL;
 421         }
 422
 423         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 424                 return 0;
 425
 426         if (hw->mac.type == fm10k_mac_vf) {
 427                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 428                 return -EINVAL;
 429         }
 430
 431         /* Check VMDQ queue pool number */
 432         if (vmdq_conf->nb_queue_pools >
 433                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 434                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 435                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 436                         vmdq_conf->nb_queue_pools);
 437                 return -EINVAL;
 438         }
 439
 440         return 0;
 441 }
 442
 443 static const struct fm10k_txq_ops def_txq_ops = {
 444         .reset = tx_queue_reset,
 445 };
 446
 447 static int
 448 fm10k_dev_configure(struct rte_eth_dev *dev)
 449 {
 450         int ret;
 451
 452         PMD_INIT_FUNC_TRACE();
 453
 454         /* multipe queue mode checking */
 455         ret  = fm10k_check_mq_mode(dev);
 456         if (ret != 0) {
 457                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 458                             ret);
 459                 return ret;
 460         }
 461
 462         dev->data->scattered_rx = 0;
 463
 464         return 0;
 465 }
 466
 467 static void
 468 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 469 {
 470         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 471         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 472         uint32_t i;
 473
 474         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 475
 476         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 477                 if (!vmdq_conf->pool_map[i].pools)
 478                         continue;
 479                 fm10k_mbx_lock(hw);
 480                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 481                 fm10k_mbx_unlock(hw);
 482         }
 483 }
 484
 485 static void
 486 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 487 {
 488         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 489
 490         /* Add default mac address */
 491         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 492                 MAIN_VSI_POOL_NUMBER);
 493 }
 494
 495 static void
 496 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 497 {
 498         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 499         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 500         uint32_t mrqc, *key, i, reta, j;
 501         uint64_t hf;
 502
 503 #define RSS_KEY_SIZE 40
 504         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 505                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 506                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 507                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 508                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 509                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 510         };
 511
 512         if (dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 513                 dev_conf->rx_adv_conf.rss_conf.rss_hf == 0) {
 514                 FM10K_WRITE_REG(hw, FM10K_MRQC(0), 0);
 515                 return;
 516         }
 517
 518         /* random key is rss_intel_key (default) or user provided (rss_key) */
 519         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 520                 key = (uint32_t *)rss_intel_key;
 521         else
 522                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 523
 524         /* Now fill our hash function seeds, 4 bytes at a time */
 525         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 526                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 527
 528         /*
 529          * Fill in redirection table
 530          * The byte-swap is needed because NIC registers are in
 531          * little-endian order.
 532          */
 533         reta = 0;
 534         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 535                 if (j == dev->data->nb_rx_queues)
 536                         j = 0;
 537                 reta = (reta << CHAR_BIT) | j;
 538                 if ((i & 3) == 3)
 539                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 540                                         rte_bswap32(reta));
 541         }
 542
 543         /*
 544          * Generate RSS hash based on packet types, TCP/UDP
 545          * port numbers and/or IPv4/v6 src and dst addresses
 546          */
 547         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 548         mrqc = 0;
 549         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 550         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 551         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 552         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 553         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 554         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 555         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 556         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 557         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 558
 559         if (mrqc == 0) {
 560                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 561                         "supported", hf);
 562                 return;
 563         }
 564
 565         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 566 }
 567
 568 static void
 569 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 570 {
 571         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 572         uint32_t i;
 573
 574         for (i = 0; i < nb_lport_new; i++) {
 575                 /* Set unicast mode by default. App can change
 576                  * to other mode in other API func.
 577                  */
 578                 fm10k_mbx_lock(hw);
 579                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 580                         FM10K_XCAST_MODE_NONE);
 581                 fm10k_mbx_unlock(hw);
 582         }
 583 }
 584
 585 static void
 586 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 587 {
 588         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 589         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 590         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 591         struct fm10k_macvlan_filter_info *macvlan;
 592         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 593         uint16_t nb_lport_new;
 594
 595         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 596         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 597
 598         fm10k_dev_rss_configure(dev);
 599
 600         /* only PF supports VMDQ */
 601         if (hw->mac.type != fm10k_mac_pf)
 602                 return;
 603
 604         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 605                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 606
 607         /* no pool number change, no need to update logic port and VLAN/MAC */
 608         if (macvlan->nb_queue_pools == nb_queue_pools)
 609                 return;
 610
 611         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 612         fm10k_dev_logic_port_update(dev, nb_lport_new);
 613
 614         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 615         memset(dev->data->mac_addrs, 0,
 616                 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 617         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
 618                 &dev->data->mac_addrs[0]);
 619         memset(macvlan, 0, sizeof(*macvlan));
 620         macvlan->nb_queue_pools = nb_queue_pools;
 621
 622         if (nb_queue_pools)
 623                 fm10k_dev_vmdq_rx_configure(dev);
 624         else
 625                 fm10k_dev_pf_main_vsi_reset(dev);
 626 }
 627
 628 static int
 629 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 630 {
 631         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 632         int i, ret;
 633         struct fm10k_tx_queue *txq;
 634         uint64_t base_addr;
 635         uint32_t size;
 636
 637         /* Disable TXINT to avoid possible interrupt */
 638         for (i = 0; i < hw->mac.max_queues; i++)
 639                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 640                                 3 << FM10K_TXINT_TIMER_SHIFT);
 641
 642         /* Setup TX queue */
 643         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 644                 txq = dev->data->tx_queues[i];
 645                 base_addr = txq->hw_ring_phys_addr;
 646                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 647
 648                 /* disable queue to avoid issues while updating state */
 649                 ret = tx_queue_disable(hw, i);
 650                 if (ret) {
 651                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 652                         return -1;
 653                 }
 654                 /* Enable use of FTAG bit in TX descriptor, PFVTCTL
 655                  * register is read-only for VF.
 656                  */
 657                 if (fm10k_check_ftag(dev->device->devargs)) {
 658                         if (hw->mac.type == fm10k_mac_pf) {
 659                                 FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i),
 660                                                 FM10K_PFVTCTL_FTAG_DESC_ENABLE);
 661                                 PMD_INIT_LOG(DEBUG, "FTAG mode is enabled");
 662                         } else {
 663                                 PMD_INIT_LOG(ERR, "VF FTAG is not supported.");
 664                                 return -ENOTSUP;
 665                         }
 666                 }
 667
 668                 /* set location and size for descriptor ring */
 669                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 670                                 base_addr & UINT64_LOWER_32BITS_MASK);
 671                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 672                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 673                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 674
 675                 /* assign default SGLORT for each TX queue by PF */
 676                 if (hw->mac.type == fm10k_mac_pf)
 677                         FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map);
 678         }
 679
 680         /* set up vector or scalar TX function as appropriate */
 681         fm10k_set_tx_function(dev);
 682
 683         return 0;
 684 }
 685
 686 static int
 687 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 688 {
 689         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 690         struct fm10k_macvlan_filter_info *macvlan;
 691         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
 692         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
 693         int i, ret;
 694         struct fm10k_rx_queue *rxq;
 695         uint64_t base_addr;
 696         uint32_t size;
 697         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 698         uint32_t logic_port = hw->mac.dglort_map;
 699         uint16_t buf_size;
 700         uint16_t queue_stride = 0;
 701
 702         /* enable RXINT for interrupt mode */
 703         i = 0;
 704         if (rte_intr_dp_is_en(intr_handle)) {
 705                 for (; i < dev->data->nb_rx_queues; i++) {
 706                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(pdev, i));
 707                         if (hw->mac.type == fm10k_mac_pf)
 708                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
 709                                         FM10K_ITR_AUTOMASK |
 710                                         FM10K_ITR_MASK_CLEAR);
 711                         else
 712                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
 713                                         FM10K_ITR_AUTOMASK |
 714                                         FM10K_ITR_MASK_CLEAR);
 715                 }
 716         }
 717         /* Disable other RXINT to avoid possible interrupt */
 718         for (; i < hw->mac.max_queues; i++)
 719                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 720                         3 << FM10K_RXINT_TIMER_SHIFT);
 721
 722         /* Setup RX queues */
 723         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 724                 rxq = dev->data->rx_queues[i];
 725                 base_addr = rxq->hw_ring_phys_addr;
 726                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 727
 728                 /* disable queue to avoid issues while updating state */
 729                 ret = rx_queue_disable(hw, i);
 730                 if (ret) {
 731                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 732                         return -1;
 733                 }
 734
 735                 /* Setup the Base and Length of the Rx Descriptor Ring */
 736                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 737                                 base_addr & UINT64_LOWER_32BITS_MASK);
 738                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 739                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 740                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 741
 742                 /* Configure the Rx buffer size for one buff without split */
 743                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 744                         RTE_PKTMBUF_HEADROOM);
 745                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 746                  * reserved for this purpose, and the worst case could be 511B.
 747                  * But SRR reg assumes all buffers have the same size. In order
 748                  * to fill the gap, we'll have to consider the worst case and
 749                  * assume 512B is reserved. If we don't do so, it's possible
 750                  * for HW to overwrite data to next mbuf.
 751                  */
 752                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 753
 754                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 755                                 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
 756                                 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
 757
 758                 /* It adds dual VLAN length for supporting dual VLAN */
 759                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 760                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 761                         rxq->offloads & DEV_RX_OFFLOAD_SCATTER) {
 762                         uint32_t reg;
 763                         dev->data->scattered_rx = 1;
 764                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 765                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 766                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 767                 }
 768
 769                 /* Enable drop on empty, it's RO for VF */
 770                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 771                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 772
 773                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 774                 FM10K_WRITE_FLUSH(hw);
 775         }
 776
 777         /* Configure VMDQ/RSS if applicable */
 778         fm10k_dev_mq_rx_configure(dev);
 779
 780         /* Decide the best RX function */
 781         fm10k_set_rx_function(dev);
 782
 783         /* update RX_SGLORT for loopback suppress*/
 784         if (hw->mac.type != fm10k_mac_pf)
 785                 return 0;
 786         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 787         if (macvlan->nb_queue_pools)
 788                 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
 789         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 790                 if (i && queue_stride && !(i % queue_stride))
 791                         logic_port++;
 792                 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
 793         }
 794
 795         return 0;
 796 }
 797
 798 static int
 799 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 800 {
 801         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 802         int err;
 803         uint32_t reg;
 804         struct fm10k_rx_queue *rxq;
 805
 806         PMD_INIT_FUNC_TRACE();
 807
 808         rxq = dev->data->rx_queues[rx_queue_id];
 809         err = rx_queue_reset(rxq);
 810         if (err == -ENOMEM) {
 811                 PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 812                 return err;
 813         } else if (err == -EINVAL) {
 814                 PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 815                         " %d", err);
 816                 return err;
 817         }
 818
 819         /* Setup the HW Rx Head and Tail Descriptor Pointers
 820          * Note: this must be done AFTER the queue is enabled on real
 821          * hardware, but BEFORE the queue is enabled when using the
 822          * emulation platform. Do it in both places for now and remove
 823          * this comment and the following two register writes when the
 824          * emulation platform is no longer being used.
 825          */
 826         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 827         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 828
 829         /* Set PF ownership flag for PF devices */
 830         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 831         if (hw->mac.type == fm10k_mac_pf)
 832                 reg |= FM10K_RXQCTL_PF;
 833         reg |= FM10K_RXQCTL_ENABLE;
 834         /* enable RX queue */
 835         FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 836         FM10K_WRITE_FLUSH(hw);
 837
 838         /* Setup the HW Rx Head and Tail Descriptor Pointers
 839          * Note: this must be done AFTER the queue is enabled
 840          */
 841         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 842         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 843         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 844
 845         return 0;
 846 }
 847
 848 static int
 849 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 850 {
 851         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 852
 853         PMD_INIT_FUNC_TRACE();
 854
 855         /* Disable RX queue */
 856         rx_queue_disable(hw, rx_queue_id);
 857
 858         /* Free mbuf and clean HW ring */
 859         rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 860         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 861
 862         return 0;
 863 }
 864
 865 static int
 866 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 867 {
 868         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 869         /** @todo - this should be defined in the shared code */
 870 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 871         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 872         struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 873
 874         PMD_INIT_FUNC_TRACE();
 875
 876         q->ops->reset(q);
 877
 878         /* reset head and tail pointers */
 879         FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 880         FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 881
 882         /* enable TX queue */
 883         FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 884                                 FM10K_TXDCTL_ENABLE | txdctl);
 885         FM10K_WRITE_FLUSH(hw);
 886         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 887
 888         return 0;
 889 }
 890
 891 static int
 892 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 893 {
 894         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 895
 896         PMD_INIT_FUNC_TRACE();
 897
 898         tx_queue_disable(hw, tx_queue_id);
 899         tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 900         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 901
 902         return 0;
 903 }
 904
 905 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 906 {
 907         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 908                 != FM10K_DGLORTMAP_NONE);
 909 }
 910
 911 static void
 912 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 913 {
 914         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 915         int status;
 916
 917         PMD_INIT_FUNC_TRACE();
 918
 919         /* Return if it didn't acquire valid glort range */
 920         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 921                 return;
 922
 923         fm10k_mbx_lock(hw);
 924         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 925                                 FM10K_XCAST_MODE_PROMISC);
 926         fm10k_mbx_unlock(hw);
 927
 928         if (status != FM10K_SUCCESS)
 929                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 930 }
 931
 932 static void
 933 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 934 {
 935         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 936         uint8_t mode;
 937         int status;
 938
 939         PMD_INIT_FUNC_TRACE();
 940
 941         /* Return if it didn't acquire valid glort range */
 942         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 943                 return;
 944
 945         if (dev->data->all_multicast == 1)
 946                 mode = FM10K_XCAST_MODE_ALLMULTI;
 947         else
 948                 mode = FM10K_XCAST_MODE_NONE;
 949
 950         fm10k_mbx_lock(hw);
 951         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 952                                 mode);
 953         fm10k_mbx_unlock(hw);
 954
 955         if (status != FM10K_SUCCESS)
 956                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 957 }
 958
 959 static void
 960 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 961 {
 962         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 963         int status;
 964
 965         PMD_INIT_FUNC_TRACE();
 966
 967         /* Return if it didn't acquire valid glort range */
 968         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 969                 return;
 970
 971         /* If promiscuous mode is enabled, it doesn't make sense to enable
 972          * allmulticast and disable promiscuous since fm10k only can select
 973          * one of the modes.
 974          */
 975         if (dev->data->promiscuous) {
 976                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 977                         "needn't enable allmulticast");
 978                 return;
 979         }
 980
 981         fm10k_mbx_lock(hw);
 982         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 983                                 FM10K_XCAST_MODE_ALLMULTI);
 984         fm10k_mbx_unlock(hw);
 985
 986         if (status != FM10K_SUCCESS)
 987                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 988 }
 989
 990 static void
 991 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
 992 {
 993         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 994         int status;
 995
 996         PMD_INIT_FUNC_TRACE();
 997
 998         /* Return if it didn't acquire valid glort range */
 999         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1000                 return;
1001
1002         if (dev->data->promiscuous) {
1003                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1004                         "since promisc mode is enabled");
1005                 return;
1006         }
1007
1008         fm10k_mbx_lock(hw);
1009         /* Change mode to unicast mode */
1010         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1011                                 FM10K_XCAST_MODE_NONE);
1012         fm10k_mbx_unlock(hw);
1013
1014         if (status != FM10K_SUCCESS)
1015                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1016 }
1017
1018 static void
1019 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1020 {
1021         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1022         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1023         uint16_t nb_queue_pools;
1024         struct fm10k_macvlan_filter_info *macvlan;
1025
1026         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1027         nb_queue_pools = macvlan->nb_queue_pools;
1028         pool_len = nb_queue_pools ? rte_fls_u32(nb_queue_pools - 1) : 0;
1029         rss_len = rte_fls_u32(dev->data->nb_rx_queues - 1) - pool_len;
1030
1031         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1032         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1033         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1034                         hw->mac.dglort_map;
1035         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1036         /* Configure VMDQ/RSS DGlort Decoder */
1037         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1038
1039         /* Flow Director configurations, only queue number is valid. */
1040         dglortdec = rte_fls_u32(dev->data->nb_rx_queues - 1);
1041         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1042                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1043         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1044         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1045
1046         /* Invalidate all other GLORT entries */
1047         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1048                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1049                                 FM10K_DGLORTMAP_NONE);
1050 }
1051
1052 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1053 static int
1054 fm10k_dev_start(struct rte_eth_dev *dev)
1055 {
1056         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1057         int i, diag;
1058
1059         PMD_INIT_FUNC_TRACE();
1060
1061         /* stop, init, then start the hw */
1062         diag = fm10k_stop_hw(hw);
1063         if (diag != FM10K_SUCCESS) {
1064                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1065                 return -EIO;
1066         }
1067
1068         diag = fm10k_init_hw(hw);
1069         if (diag != FM10K_SUCCESS) {
1070                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1071                 return -EIO;
1072         }
1073
1074         diag = fm10k_start_hw(hw);
1075         if (diag != FM10K_SUCCESS) {
1076                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1077                 return -EIO;
1078         }
1079
1080         diag = fm10k_dev_tx_init(dev);
1081         if (diag) {
1082                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1083                 return diag;
1084         }
1085
1086         if (fm10k_dev_rxq_interrupt_setup(dev))
1087                 return -EIO;
1088
1089         diag = fm10k_dev_rx_init(dev);
1090         if (diag) {
1091                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1092                 return diag;
1093         }
1094
1095         if (hw->mac.type == fm10k_mac_pf)
1096                 fm10k_dev_dglort_map_configure(dev);
1097
1098         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1099                 struct fm10k_rx_queue *rxq;
1100                 rxq = dev->data->rx_queues[i];
1101
1102                 if (rxq->rx_deferred_start)
1103                         continue;
1104                 diag = fm10k_dev_rx_queue_start(dev, i);
1105                 if (diag != 0) {
1106                         int j;
1107                         for (j = 0; j < i; ++j)
1108                                 rx_queue_clean(dev->data->rx_queues[j]);
1109                         return diag;
1110                 }
1111         }
1112
1113         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1114                 struct fm10k_tx_queue *txq;
1115                 txq = dev->data->tx_queues[i];
1116
1117                 if (txq->tx_deferred_start)
1118                         continue;
1119                 diag = fm10k_dev_tx_queue_start(dev, i);
1120                 if (diag != 0) {
1121                         int j;
1122                         for (j = 0; j < i; ++j)
1123                                 tx_queue_clean(dev->data->tx_queues[j]);
1124                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1125                                 rx_queue_clean(dev->data->rx_queues[j]);
1126                         return diag;
1127                 }
1128         }
1129
1130         /* Update default vlan when not in VMDQ mode */
1131         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1132                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1133
1134         fm10k_link_update(dev, 0);
1135
1136         return 0;
1137 }
1138
1139 static void
1140 fm10k_dev_stop(struct rte_eth_dev *dev)
1141 {
1142         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1143         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1144         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
1145         int i;
1146
1147         PMD_INIT_FUNC_TRACE();
1148
1149         if (dev->data->tx_queues)
1150                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1151                         fm10k_dev_tx_queue_stop(dev, i);
1152
1153         if (dev->data->rx_queues)
1154                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1155                         fm10k_dev_rx_queue_stop(dev, i);
1156
1157         /* Disable datapath event */
1158         if (rte_intr_dp_is_en(intr_handle)) {
1159                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1160                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1161                                 3 << FM10K_RXINT_TIMER_SHIFT);
1162                         if (hw->mac.type == fm10k_mac_pf)
1163                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
1164                                         FM10K_ITR_MASK_SET);
1165                         else
1166                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
1167                                         FM10K_ITR_MASK_SET);
1168                 }
1169         }
1170         /* Clean datapath event and queue/vec mapping */
1171         rte_intr_efd_disable(intr_handle);
1172         rte_free(intr_handle->intr_vec);
1173         intr_handle->intr_vec = NULL;
1174 }
1175
1176 static void
1177 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1178 {
1179         int i;
1180
1181         PMD_INIT_FUNC_TRACE();
1182
1183         if (dev->data->tx_queues) {
1184                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1185                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1186
1187                         tx_queue_free(txq);
1188                 }
1189         }
1190
1191         if (dev->data->rx_queues) {
1192                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1193                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1194         }
1195 }
1196
1197 static void
1198 fm10k_dev_close(struct rte_eth_dev *dev)
1199 {
1200         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1201
1202         PMD_INIT_FUNC_TRACE();
1203
1204         fm10k_mbx_lock(hw);
1205         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1206                 MAX_LPORT_NUM, false);
1207         fm10k_mbx_unlock(hw);
1208
1209         /* allow 100ms for device to quiesce */
1210         rte_delay_us(FM10K_SWITCH_QUIESCE_US);
1211
1212         /* Stop mailbox service first */
1213         fm10k_close_mbx_service(hw);
1214         fm10k_dev_stop(dev);
1215         fm10k_dev_queue_release(dev);
1216         fm10k_stop_hw(hw);
1217 }
1218
1219 static int
1220 fm10k_link_update(struct rte_eth_dev *dev,
1221         __rte_unused int wait_to_complete)
1222 {
1223         struct fm10k_dev_info *dev_info =
1224                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1225         PMD_INIT_FUNC_TRACE();
1226
1227         dev->data->dev_link.link_speed  = ETH_SPEED_NUM_50G;
1228         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1229         dev->data->dev_link.link_status =
1230                 dev_info->sm_down ? ETH_LINK_DOWN : ETH_LINK_UP;
1231         dev->data->dev_link.link_autoneg = ETH_LINK_FIXED;
1232
1233         return 0;
1234 }
1235
1236 static int fm10k_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1237         struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit)
1238 {
1239         unsigned i, q;
1240         unsigned count = 0;
1241
1242         if (xstats_names != NULL) {
1243                 /* Note: limit checked in rte_eth_xstats_names() */
1244
1245                 /* Global stats */
1246                 for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1247                         snprintf(xstats_names[count].name,
1248                                 sizeof(xstats_names[count].name),
1249                                 "%s", fm10k_hw_stats_strings[count].name);
1250                         count++;
1251                 }
1252
1253                 /* PF queue stats */
1254                 for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1255                         for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1256                                 snprintf(xstats_names[count].name,
1257                                         sizeof(xstats_names[count].name),
1258                                         "rx_q%u_%s", q,
1259                                         fm10k_hw_stats_rx_q_strings[i].name);
1260                                 count++;
1261                         }
1262                         for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1263                                 snprintf(xstats_names[count].name,
1264                                         sizeof(xstats_names[count].name),
1265                                         "tx_q%u_%s", q,
1266                                         fm10k_hw_stats_tx_q_strings[i].name);
1267                                 count++;
1268                         }
1269                 }
1270         }
1271         return FM10K_NB_XSTATS;
1272 }
1273
1274 static int
1275 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1276                  unsigned n)
1277 {
1278         struct fm10k_hw_stats *hw_stats =
1279                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1280         unsigned i, q, count = 0;
1281
1282         if (n < FM10K_NB_XSTATS)
1283                 return FM10K_NB_XSTATS;
1284
1285         /* Global stats */
1286         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1287                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1288                         fm10k_hw_stats_strings[count].offset);
1289                 xstats[count].id = count;
1290                 count++;
1291         }
1292
1293         /* PF queue stats */
1294         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1295                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1296                         xstats[count].value =
1297                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1298                                 fm10k_hw_stats_rx_q_strings[i].offset);
1299                         xstats[count].id = count;
1300                         count++;
1301                 }
1302                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1303                         xstats[count].value =
1304                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1305                                 fm10k_hw_stats_tx_q_strings[i].offset);
1306                         xstats[count].id = count;
1307                         count++;
1308                 }
1309         }
1310
1311         return FM10K_NB_XSTATS;
1312 }
1313
1314 static int
1315 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1316 {
1317         uint64_t ipackets, opackets, ibytes, obytes, imissed;
1318         struct fm10k_hw *hw =
1319                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1320         struct fm10k_hw_stats *hw_stats =
1321                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1322         int i;
1323
1324         PMD_INIT_FUNC_TRACE();
1325
1326         fm10k_update_hw_stats(hw, hw_stats);
1327
1328         ipackets = opackets = ibytes = obytes = imissed = 0;
1329         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1330                 (i < hw->mac.max_queues); ++i) {
1331                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1332                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1333                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1334                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1335                 stats->q_errors[i]   = hw_stats->q[i].rx_drops.count;
1336                 ipackets += stats->q_ipackets[i];
1337                 opackets += stats->q_opackets[i];
1338                 ibytes   += stats->q_ibytes[i];
1339                 obytes   += stats->q_obytes[i];
1340                 imissed  += stats->q_errors[i];
1341         }
1342         stats->ipackets = ipackets;
1343         stats->opackets = opackets;
1344         stats->ibytes = ibytes;
1345         stats->obytes = obytes;
1346         stats->imissed = imissed;
1347         return 0;
1348 }
1349
1350 static void
1351 fm10k_stats_reset(struct rte_eth_dev *dev)
1352 {
1353         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1354         struct fm10k_hw_stats *hw_stats =
1355                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1356
1357         PMD_INIT_FUNC_TRACE();
1358
1359         memset(hw_stats, 0, sizeof(*hw_stats));
1360         fm10k_rebind_hw_stats(hw, hw_stats);
1361 }
1362
1363 static void
1364 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1365         struct rte_eth_dev_info *dev_info)
1366 {
1367         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1368         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1369
1370         PMD_INIT_FUNC_TRACE();
1371
1372         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1373         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1374         dev_info->max_rx_queues      = hw->mac.max_queues;
1375         dev_info->max_tx_queues      = hw->mac.max_queues;
1376         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1377         dev_info->max_hash_mac_addrs = 0;
1378         dev_info->max_vfs            = pdev->max_vfs;
1379         dev_info->vmdq_pool_base     = 0;
1380         dev_info->vmdq_queue_base    = 0;
1381         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1382         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1383         dev_info->rx_queue_offload_capa = fm10k_get_rx_queue_offloads_capa(dev);
1384         dev_info->rx_offload_capa = fm10k_get_rx_port_offloads_capa(dev) |
1385                                     dev_info->rx_queue_offload_capa;
1386         dev_info->tx_queue_offload_capa = fm10k_get_tx_queue_offloads_capa(dev);
1387         dev_info->tx_offload_capa = fm10k_get_tx_port_offloads_capa(dev) |
1388                                     dev_info->tx_queue_offload_capa;
1389
1390         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1391         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1392
1393         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1394                 .rx_thresh = {
1395                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1396                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1397                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1398                 },
1399                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1400                 .rx_drop_en = 0,
1401                 .offloads = 0,
1402         };
1403
1404         dev_info->default_txconf = (struct rte_eth_txconf) {
1405                 .tx_thresh = {
1406                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1407                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1408                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1409                 },
1410                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1411                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1412                 .offloads = 0,
1413         };
1414
1415         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1416                 .nb_max = FM10K_MAX_RX_DESC,
1417                 .nb_min = FM10K_MIN_RX_DESC,
1418                 .nb_align = FM10K_MULT_RX_DESC,
1419         };
1420
1421         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1422                 .nb_max = FM10K_MAX_TX_DESC,
1423                 .nb_min = FM10K_MIN_TX_DESC,
1424                 .nb_align = FM10K_MULT_TX_DESC,
1425                 .nb_seg_max = FM10K_TX_MAX_SEG,
1426                 .nb_mtu_seg_max = FM10K_TX_MAX_MTU_SEG,
1427         };
1428
1429         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
1430                         ETH_LINK_SPEED_10G | ETH_LINK_SPEED_25G |
1431                         ETH_LINK_SPEED_40G | ETH_LINK_SPEED_100G;
1432 }
1433
1434 #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
1435 static const uint32_t *
1436 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1437 {
1438         if (dev->rx_pkt_burst == fm10k_recv_pkts ||
1439             dev->rx_pkt_burst == fm10k_recv_scattered_pkts) {
1440                 static uint32_t ptypes[] = {
1441                         /* refers to rx_desc_to_ol_flags() */
1442                         RTE_PTYPE_L2_ETHER,
1443                         RTE_PTYPE_L3_IPV4,
1444                         RTE_PTYPE_L3_IPV4_EXT,
1445                         RTE_PTYPE_L3_IPV6,
1446                         RTE_PTYPE_L3_IPV6_EXT,
1447                         RTE_PTYPE_L4_TCP,
1448                         RTE_PTYPE_L4_UDP,
1449                         RTE_PTYPE_UNKNOWN
1450                 };
1451
1452                 return ptypes;
1453         } else if (dev->rx_pkt_burst == fm10k_recv_pkts_vec ||
1454                    dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec) {
1455                 static uint32_t ptypes_vec[] = {
1456                         /* refers to fm10k_desc_to_pktype_v() */
1457                         RTE_PTYPE_L3_IPV4,
1458                         RTE_PTYPE_L3_IPV4_EXT,
1459                         RTE_PTYPE_L3_IPV6,
1460                         RTE_PTYPE_L3_IPV6_EXT,
1461                         RTE_PTYPE_L4_TCP,
1462                         RTE_PTYPE_L4_UDP,
1463                         RTE_PTYPE_TUNNEL_GENEVE,
1464                         RTE_PTYPE_TUNNEL_NVGRE,
1465                         RTE_PTYPE_TUNNEL_VXLAN,
1466                         RTE_PTYPE_TUNNEL_GRE,
1467                         RTE_PTYPE_UNKNOWN
1468                 };
1469
1470                 return ptypes_vec;
1471         }
1472
1473         return NULL;
1474 }
1475 #else
1476 static const uint32_t *
1477 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1478 {
1479         return NULL;
1480 }
1481 #endif
1482
1483 static int
1484 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1485 {
1486         s32 result;
1487         uint16_t mac_num = 0;
1488         uint32_t vid_idx, vid_bit, mac_index;
1489         struct fm10k_hw *hw;
1490         struct fm10k_macvlan_filter_info *macvlan;
1491         struct rte_eth_dev_data *data = dev->data;
1492
1493         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1494         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1495
1496         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1497                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1498                 return -EINVAL;
1499         }
1500
1501         if (vlan_id > ETH_VLAN_ID_MAX) {
1502                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1503                 return -EINVAL;
1504         }
1505
1506         vid_idx = FM10K_VFTA_IDX(vlan_id);
1507         vid_bit = FM10K_VFTA_BIT(vlan_id);
1508         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1509         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1510                 return 0;
1511         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1512         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1513                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1514                         "in the VLAN filter table");
1515                 return -EINVAL;
1516         }
1517
1518         fm10k_mbx_lock(hw);
1519         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1520         fm10k_mbx_unlock(hw);
1521         if (result != FM10K_SUCCESS) {
1522                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1523                 return -EIO;
1524         }
1525
1526         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1527                         (result == FM10K_SUCCESS); mac_index++) {
1528                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1529                         continue;
1530                 if (mac_num > macvlan->mac_num - 1) {
1531                         PMD_INIT_LOG(ERR, "MAC address number "
1532                                         "not match");
1533                         break;
1534                 }
1535                 fm10k_mbx_lock(hw);
1536                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1537                         data->mac_addrs[mac_index].addr_bytes,
1538                         vlan_id, on, 0);
1539                 fm10k_mbx_unlock(hw);
1540                 mac_num++;
1541         }
1542         if (result != FM10K_SUCCESS) {
1543                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1544                 return -EIO;
1545         }
1546
1547         if (on) {
1548                 macvlan->vlan_num++;
1549                 macvlan->vfta[vid_idx] |= vid_bit;
1550         } else {
1551                 macvlan->vlan_num--;
1552                 macvlan->vfta[vid_idx] &= ~vid_bit;
1553         }
1554         return 0;
1555 }
1556
1557 static int
1558 fm10k_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1559 {
1560         if (mask & ETH_VLAN_STRIP_MASK) {
1561                 if (!(dev->data->dev_conf.rxmode.offloads &
1562                         DEV_RX_OFFLOAD_VLAN_STRIP))
1563                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1564                                         "always on in fm10k");
1565         }
1566
1567         if (mask & ETH_VLAN_EXTEND_MASK) {
1568                 if (dev->data->dev_conf.rxmode.offloads &
1569                         DEV_RX_OFFLOAD_VLAN_EXTEND)
1570                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1571                                         "supported in fm10k");
1572         }
1573
1574         if (mask & ETH_VLAN_FILTER_MASK) {
1575                 if (!(dev->data->dev_conf.rxmode.offloads &
1576                         DEV_RX_OFFLOAD_VLAN_FILTER))
1577                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1578         }
1579
1580         return 0;
1581 }
1582
1583 /* Add/Remove a MAC address, and update filters to main VSI */
1584 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1585                 const u8 *mac, bool add, uint32_t pool)
1586 {
1587         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1588         struct fm10k_macvlan_filter_info *macvlan;
1589         uint32_t i, j, k;
1590
1591         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1592
1593         if (pool != MAIN_VSI_POOL_NUMBER) {
1594                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1595                         "mac to pool %u", pool);
1596                 return;
1597         }
1598         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1599                 if (!macvlan->vfta[j])
1600                         continue;
1601                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1602                         if (!(macvlan->vfta[j] & (1 << k)))
1603                                 continue;
1604                         if (i + 1 > macvlan->vlan_num) {
1605                                 PMD_INIT_LOG(ERR, "vlan number not match");
1606                                 return;
1607                         }
1608                         fm10k_mbx_lock(hw);
1609                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1610                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1611                         fm10k_mbx_unlock(hw);
1612                         i++;
1613                 }
1614         }
1615 }
1616
1617 /* Add/Remove a MAC address, and update filters to VMDQ */
1618 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1619                 const u8 *mac, bool add, uint32_t pool)
1620 {
1621         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1622         struct fm10k_macvlan_filter_info *macvlan;
1623         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1624         uint32_t i;
1625
1626         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1627         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1628
1629         if (pool > macvlan->nb_queue_pools) {
1630                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1631                         " Max pool is %u",
1632                         pool, macvlan->nb_queue_pools);
1633                 return;
1634         }
1635         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1636                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1637                         continue;
1638                 fm10k_mbx_lock(hw);
1639                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1640                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1641                 fm10k_mbx_unlock(hw);
1642         }
1643 }
1644
1645 /* Add/Remove a MAC address, and update filters */
1646 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1647                 const u8 *mac, bool add, uint32_t pool)
1648 {
1649         struct fm10k_macvlan_filter_info *macvlan;
1650
1651         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1652
1653         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1654                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1655         else
1656                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1657
1658         if (add)
1659                 macvlan->mac_num++;
1660         else
1661                 macvlan->mac_num--;
1662 }
1663
1664 /* Add a MAC address, and update filters */
1665 static int
1666 fm10k_macaddr_add(struct rte_eth_dev *dev,
1667                 struct ether_addr *mac_addr,
1668                 uint32_t index,
1669                 uint32_t pool)
1670 {
1671         struct fm10k_macvlan_filter_info *macvlan;
1672
1673         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1674         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1675         macvlan->mac_vmdq_id[index] = pool;
1676         return 0;
1677 }
1678
1679 /* Remove a MAC address, and update filters */
1680 static void
1681 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1682 {
1683         struct rte_eth_dev_data *data = dev->data;
1684         struct fm10k_macvlan_filter_info *macvlan;
1685
1686         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1687         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1688                         FALSE, macvlan->mac_vmdq_id[index]);
1689         macvlan->mac_vmdq_id[index] = 0;
1690 }
1691
1692 static inline int
1693 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1694 {
1695         if ((request < min) || (request > max) || ((request % mult) != 0))
1696                 return -1;
1697         else
1698                 return 0;
1699 }
1700
1701
1702 static inline int
1703 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1704 {
1705         if ((request < min) || (request > max) || ((div % request) != 0))
1706                 return -1;
1707         else
1708                 return 0;
1709 }
1710
1711 static inline int
1712 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1713 {
1714         uint16_t rx_free_thresh;
1715
1716         if (conf->rx_free_thresh == 0)
1717                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1718         else
1719                 rx_free_thresh = conf->rx_free_thresh;
1720
1721         /* make sure the requested threshold satisfies the constraints */
1722         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1723                         FM10K_RX_FREE_THRESH_MAX(q),
1724                         FM10K_RX_FREE_THRESH_DIV(q),
1725                         rx_free_thresh)) {
1726                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1727                         "less than or equal to %u, "
1728                         "greater than or equal to %u, "
1729                         "and a divisor of %u",
1730                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1731                         FM10K_RX_FREE_THRESH_MIN(q),
1732                         FM10K_RX_FREE_THRESH_DIV(q));
1733                 return -EINVAL;
1734         }
1735
1736         q->alloc_thresh = rx_free_thresh;
1737         q->drop_en = conf->rx_drop_en;
1738         q->rx_deferred_start = conf->rx_deferred_start;
1739
1740         return 0;
1741 }
1742
1743 /*
1744  * Hardware requires specific alignment for Rx packet buffers. At
1745  * least one of the following two conditions must be satisfied.
1746  *  1. Address is 512B aligned
1747  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1748  *
1749  * As such, the driver may need to adjust the DMA address within the
1750  * buffer by up to 512B.
1751  *
1752  * return 1 if the element size is valid, otherwise return 0.
1753  */
1754 static int
1755 mempool_element_size_valid(struct rte_mempool *mp)
1756 {
1757         uint32_t min_size;
1758
1759         /* elt_size includes mbuf header and headroom */
1760         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1761                         RTE_PKTMBUF_HEADROOM;
1762
1763         /* account for up to 512B of alignment */
1764         min_size -= FM10K_RX_DATABUF_ALIGN;
1765
1766         /* sanity check for overflow */
1767         if (min_size > mp->elt_size)
1768                 return 0;
1769
1770         /* size is valid */
1771         return 1;
1772 }
1773
1774 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev)
1775 {
1776         RTE_SET_USED(dev);
1777
1778         return (uint64_t)(DEV_RX_OFFLOAD_SCATTER);
1779 }
1780
1781 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev)
1782 {
1783         RTE_SET_USED(dev);
1784
1785         return  (uint64_t)(DEV_RX_OFFLOAD_VLAN_STRIP  |
1786                            DEV_RX_OFFLOAD_VLAN_FILTER |
1787                            DEV_RX_OFFLOAD_IPV4_CKSUM  |
1788                            DEV_RX_OFFLOAD_UDP_CKSUM   |
1789                            DEV_RX_OFFLOAD_TCP_CKSUM   |
1790                            DEV_RX_OFFLOAD_JUMBO_FRAME |
1791                            DEV_RX_OFFLOAD_HEADER_SPLIT);
1792 }
1793
1794 static int
1795 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1796         uint16_t nb_desc, unsigned int socket_id,
1797         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1798 {
1799         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1800         struct fm10k_dev_info *dev_info =
1801                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1802         struct fm10k_rx_queue *q;
1803         const struct rte_memzone *mz;
1804         uint64_t offloads;
1805
1806         PMD_INIT_FUNC_TRACE();
1807
1808         offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads;
1809
1810         /* make sure the mempool element size can account for alignment. */
1811         if (!mempool_element_size_valid(mp)) {
1812                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1813                 return -EINVAL;
1814         }
1815
1816         /* make sure a valid number of descriptors have been requested */
1817         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1818                                 FM10K_MULT_RX_DESC, nb_desc)) {
1819                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1820                         "less than or equal to %"PRIu32", "
1821                         "greater than or equal to %u, "
1822                         "and a multiple of %u",
1823                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1824                         FM10K_MULT_RX_DESC);
1825                 return -EINVAL;
1826         }
1827
1828         /*
1829          * if this queue existed already, free the associated memory. The
1830          * queue cannot be reused in case we need to allocate memory on
1831          * different socket than was previously used.
1832          */
1833         if (dev->data->rx_queues[queue_id] != NULL) {
1834                 rx_queue_free(dev->data->rx_queues[queue_id]);
1835                 dev->data->rx_queues[queue_id] = NULL;
1836         }
1837
1838         /* allocate memory for the queue structure */
1839         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1840                                 socket_id);
1841         if (q == NULL) {
1842                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1843                 return -ENOMEM;
1844         }
1845
1846         /* setup queue */
1847         q->mp = mp;
1848         q->nb_desc = nb_desc;
1849         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1850         q->port_id = dev->data->port_id;
1851         q->queue_id = queue_id;
1852         q->tail_ptr = (volatile uint32_t *)
1853                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1854         q->offloads = offloads;
1855         if (handle_rxconf(q, conf))
1856                 return -EINVAL;
1857
1858         /* allocate memory for the software ring */
1859         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1860                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1861                         RTE_CACHE_LINE_SIZE, socket_id);
1862         if (q->sw_ring == NULL) {
1863                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1864                 rte_free(q);
1865                 return -ENOMEM;
1866         }
1867
1868         /*
1869          * allocate memory for the hardware descriptor ring. A memzone large
1870          * enough to hold the maximum ring size is requested to allow for
1871          * resizing in later calls to the queue setup function.
1872          */
1873         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1874                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1875                                       socket_id);
1876         if (mz == NULL) {
1877                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1878                 rte_free(q->sw_ring);
1879                 rte_free(q);
1880                 return -ENOMEM;
1881         }
1882         q->hw_ring = mz->addr;
1883         q->hw_ring_phys_addr = mz->iova;
1884
1885         /* Check if number of descs satisfied Vector requirement */
1886         if (!rte_is_power_of_2(nb_desc)) {
1887                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1888                                     "preconditions - canceling the feature for "
1889                                     "the whole port[%d]",
1890                              q->queue_id, q->port_id);
1891                 dev_info->rx_vec_allowed = false;
1892         } else
1893                 fm10k_rxq_vec_setup(q);
1894
1895         dev->data->rx_queues[queue_id] = q;
1896         return 0;
1897 }
1898
1899 static void
1900 fm10k_rx_queue_release(void *queue)
1901 {
1902         PMD_INIT_FUNC_TRACE();
1903
1904         rx_queue_free(queue);
1905 }
1906
1907 static inline int
1908 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1909 {
1910         uint16_t tx_free_thresh;
1911         uint16_t tx_rs_thresh;
1912
1913         /* constraint MACROs require that tx_free_thresh is configured
1914          * before tx_rs_thresh */
1915         if (conf->tx_free_thresh == 0)
1916                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1917         else
1918                 tx_free_thresh = conf->tx_free_thresh;
1919
1920         /* make sure the requested threshold satisfies the constraints */
1921         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1922                         FM10K_TX_FREE_THRESH_MAX(q),
1923                         FM10K_TX_FREE_THRESH_DIV(q),
1924                         tx_free_thresh)) {
1925                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1926                         "less than or equal to %u, "
1927                         "greater than or equal to %u, "
1928                         "and a divisor of %u",
1929                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1930                         FM10K_TX_FREE_THRESH_MIN(q),
1931                         FM10K_TX_FREE_THRESH_DIV(q));
1932                 return -EINVAL;
1933         }
1934
1935         q->free_thresh = tx_free_thresh;
1936
1937         if (conf->tx_rs_thresh == 0)
1938                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1939         else
1940                 tx_rs_thresh = conf->tx_rs_thresh;
1941
1942         q->tx_deferred_start = conf->tx_deferred_start;
1943
1944         /* make sure the requested threshold satisfies the constraints */
1945         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1946                         FM10K_TX_RS_THRESH_MAX(q),
1947                         FM10K_TX_RS_THRESH_DIV(q),
1948                         tx_rs_thresh)) {
1949                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1950                         "less than or equal to %u, "
1951                         "greater than or equal to %u, "
1952                         "and a divisor of %u",
1953                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1954                         FM10K_TX_RS_THRESH_MIN(q),
1955                         FM10K_TX_RS_THRESH_DIV(q));
1956                 return -EINVAL;
1957         }
1958
1959         q->rs_thresh = tx_rs_thresh;
1960
1961         return 0;
1962 }
1963
1964 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev)
1965 {
1966         RTE_SET_USED(dev);
1967
1968         return 0;
1969 }
1970
1971 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev)
1972 {
1973         RTE_SET_USED(dev);
1974
1975         return (uint64_t)(DEV_TX_OFFLOAD_VLAN_INSERT |
1976                           DEV_TX_OFFLOAD_MULTI_SEGS  |
1977                           DEV_TX_OFFLOAD_IPV4_CKSUM  |
1978                           DEV_TX_OFFLOAD_UDP_CKSUM   |
1979                           DEV_TX_OFFLOAD_TCP_CKSUM   |
1980                           DEV_TX_OFFLOAD_TCP_TSO);
1981 }
1982
1983 static int
1984 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1985         uint16_t nb_desc, unsigned int socket_id,
1986         const struct rte_eth_txconf *conf)
1987 {
1988         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1989         struct fm10k_tx_queue *q;
1990         const struct rte_memzone *mz;
1991         uint64_t offloads;
1992
1993         PMD_INIT_FUNC_TRACE();
1994
1995         offloads = conf->offloads | dev->data->dev_conf.txmode.offloads;
1996
1997         /* make sure a valid number of descriptors have been requested */
1998         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1999                                 FM10K_MULT_TX_DESC, nb_desc)) {
2000                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
2001                         "less than or equal to %"PRIu32", "
2002                         "greater than or equal to %u, "
2003                         "and a multiple of %u",
2004                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
2005                         FM10K_MULT_TX_DESC);
2006                 return -EINVAL;
2007         }
2008
2009         /*
2010          * if this queue existed already, free the associated memory. The
2011          * queue cannot be reused in case we need to allocate memory on
2012          * different socket than was previously used.
2013          */
2014         if (dev->data->tx_queues[queue_id] != NULL) {
2015                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
2016
2017                 tx_queue_free(txq);
2018                 dev->data->tx_queues[queue_id] = NULL;
2019         }
2020
2021         /* allocate memory for the queue structure */
2022         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
2023                                 socket_id);
2024         if (q == NULL) {
2025                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
2026                 return -ENOMEM;
2027         }
2028
2029         /* setup queue */
2030         q->nb_desc = nb_desc;
2031         q->port_id = dev->data->port_id;
2032         q->queue_id = queue_id;
2033         q->offloads = offloads;
2034         q->ops = &def_txq_ops;
2035         q->tail_ptr = (volatile uint32_t *)
2036                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
2037         if (handle_txconf(q, conf))
2038                 return -EINVAL;
2039
2040         /* allocate memory for the software ring */
2041         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
2042                                         nb_desc * sizeof(struct rte_mbuf *),
2043                                         RTE_CACHE_LINE_SIZE, socket_id);
2044         if (q->sw_ring == NULL) {
2045                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
2046                 rte_free(q);
2047                 return -ENOMEM;
2048         }
2049
2050         /*
2051          * allocate memory for the hardware descriptor ring. A memzone large
2052          * enough to hold the maximum ring size is requested to allow for
2053          * resizing in later calls to the queue setup function.
2054          */
2055         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
2056                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
2057                                       socket_id);
2058         if (mz == NULL) {
2059                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
2060                 rte_free(q->sw_ring);
2061                 rte_free(q);
2062                 return -ENOMEM;
2063         }
2064         q->hw_ring = mz->addr;
2065         q->hw_ring_phys_addr = mz->iova;
2066
2067         /*
2068          * allocate memory for the RS bit tracker. Enough slots to hold the
2069          * descriptor index for each RS bit needing to be set are required.
2070          */
2071         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
2072                                 ((nb_desc + 1) / q->rs_thresh) *
2073                                 sizeof(uint16_t),
2074                                 RTE_CACHE_LINE_SIZE, socket_id);
2075         if (q->rs_tracker.list == NULL) {
2076                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
2077                 rte_free(q->sw_ring);
2078                 rte_free(q);
2079                 return -ENOMEM;
2080         }
2081
2082         dev->data->tx_queues[queue_id] = q;
2083         return 0;
2084 }
2085
2086 static void
2087 fm10k_tx_queue_release(void *queue)
2088 {
2089         struct fm10k_tx_queue *q = queue;
2090         PMD_INIT_FUNC_TRACE();
2091
2092         tx_queue_free(q);
2093 }
2094
2095 static int
2096 fm10k_reta_update(struct rte_eth_dev *dev,
2097                         struct rte_eth_rss_reta_entry64 *reta_conf,
2098                         uint16_t reta_size)
2099 {
2100         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2101         uint16_t i, j, idx, shift;
2102         uint8_t mask;
2103         uint32_t reta;
2104
2105         PMD_INIT_FUNC_TRACE();
2106
2107         if (reta_size > FM10K_MAX_RSS_INDICES) {
2108                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2109                         "(%d) doesn't match the number hardware can supported "
2110                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2111                 return -EINVAL;
2112         }
2113
2114         /*
2115          * Update Redirection Table RETA[n], n=0..31. The redirection table has
2116          * 128-entries in 32 registers
2117          */
2118         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2119                 idx = i / RTE_RETA_GROUP_SIZE;
2120                 shift = i % RTE_RETA_GROUP_SIZE;
2121                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2122                                 BIT_MASK_PER_UINT32);
2123                 if (mask == 0)
2124                         continue;
2125
2126                 reta = 0;
2127                 if (mask != BIT_MASK_PER_UINT32)
2128                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2129
2130                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2131                         if (mask & (0x1 << j)) {
2132                                 if (mask != 0xF)
2133                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
2134                                 reta |= reta_conf[idx].reta[shift + j] <<
2135                                                 (CHAR_BIT * j);
2136                         }
2137                 }
2138                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2139         }
2140
2141         return 0;
2142 }
2143
2144 static int
2145 fm10k_reta_query(struct rte_eth_dev *dev,
2146                         struct rte_eth_rss_reta_entry64 *reta_conf,
2147                         uint16_t reta_size)
2148 {
2149         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2150         uint16_t i, j, idx, shift;
2151         uint8_t mask;
2152         uint32_t reta;
2153
2154         PMD_INIT_FUNC_TRACE();
2155
2156         if (reta_size < FM10K_MAX_RSS_INDICES) {
2157                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2158                         "(%d) doesn't match the number hardware can supported "
2159                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2160                 return -EINVAL;
2161         }
2162
2163         /*
2164          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2165          * 128-entries in 32 registers
2166          */
2167         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2168                 idx = i / RTE_RETA_GROUP_SIZE;
2169                 shift = i % RTE_RETA_GROUP_SIZE;
2170                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2171                                 BIT_MASK_PER_UINT32);
2172                 if (mask == 0)
2173                         continue;
2174
2175                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2176                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2177                         if (mask & (0x1 << j))
2178                                 reta_conf[idx].reta[shift + j] = ((reta >>
2179                                         CHAR_BIT * j) & UINT8_MAX);
2180                 }
2181         }
2182
2183         return 0;
2184 }
2185
2186 static int
2187 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2188         struct rte_eth_rss_conf *rss_conf)
2189 {
2190         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2191         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2192         uint32_t mrqc;
2193         uint64_t hf = rss_conf->rss_hf;
2194         int i;
2195
2196         PMD_INIT_FUNC_TRACE();
2197
2198         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2199                                 FM10K_RSSRK_ENTRIES_PER_REG))
2200                 return -EINVAL;
2201
2202         if (hf == 0)
2203                 return -EINVAL;
2204
2205         mrqc = 0;
2206         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2207         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2208         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2209         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2210         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2211         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2212         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2213         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2214         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2215
2216         /* If the mapping doesn't fit any supported, return */
2217         if (mrqc == 0)
2218                 return -EINVAL;
2219
2220         if (key != NULL)
2221                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2222                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2223
2224         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2225
2226         return 0;
2227 }
2228
2229 static int
2230 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2231         struct rte_eth_rss_conf *rss_conf)
2232 {
2233         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2234         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2235         uint32_t mrqc;
2236         uint64_t hf;
2237         int i;
2238
2239         PMD_INIT_FUNC_TRACE();
2240
2241         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2242                                 FM10K_RSSRK_ENTRIES_PER_REG))
2243                 return -EINVAL;
2244
2245         if (key != NULL)
2246                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2247                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2248
2249         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2250         hf = 0;
2251         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2252         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2253         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2254         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2255         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2256         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2257         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2258         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2259         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2260
2261         rss_conf->rss_hf = hf;
2262
2263         return 0;
2264 }
2265
2266 static void
2267 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2268 {
2269         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2270         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2271
2272         /* Bind all local non-queue interrupt to vector 0 */
2273         int_map |= FM10K_MISC_VEC_ID;
2274
2275         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2276         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2277         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2278         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2279         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2280         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2281
2282         /* Enable misc causes */
2283         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2284                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2285                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2286                                 FM10K_EIMR_ENABLE(MAILBOX) |
2287                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2288                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2289                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2290                                 FM10K_EIMR_ENABLE(VFLR));
2291
2292         /* Enable ITR 0 */
2293         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2294                                         FM10K_ITR_MASK_CLEAR);
2295         FM10K_WRITE_FLUSH(hw);
2296 }
2297
2298 static void
2299 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2300 {
2301         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2302         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2303
2304         int_map |= FM10K_MISC_VEC_ID;
2305
2306         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2307         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2308         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2309         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2310         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2311         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2312
2313         /* Disable misc causes */
2314         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2315                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2316                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2317                                 FM10K_EIMR_DISABLE(MAILBOX) |
2318                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2319                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2320                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2321                                 FM10K_EIMR_DISABLE(VFLR));
2322
2323         /* Disable ITR 0 */
2324         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2325         FM10K_WRITE_FLUSH(hw);
2326 }
2327
2328 static void
2329 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2330 {
2331         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2332         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2333
2334         /* Bind all local non-queue interrupt to vector 0 */
2335         int_map |= FM10K_MISC_VEC_ID;
2336
2337         /* Only INT 0 available, other 15 are reserved. */
2338         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2339
2340         /* Enable ITR 0 */
2341         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2342                                         FM10K_ITR_MASK_CLEAR);
2343         FM10K_WRITE_FLUSH(hw);
2344 }
2345
2346 static void
2347 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2348 {
2349         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2350         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2351
2352         int_map |= FM10K_MISC_VEC_ID;
2353
2354         /* Only INT 0 available, other 15 are reserved. */
2355         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2356
2357         /* Disable ITR 0 */
2358         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2359         FM10K_WRITE_FLUSH(hw);
2360 }
2361
2362 static int
2363 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2364 {
2365         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2366         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2367
2368         /* Enable ITR */
2369         if (hw->mac.type == fm10k_mac_pf)
2370                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2371                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2372         else
2373                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2374                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2375         rte_intr_enable(&pdev->intr_handle);
2376         return 0;
2377 }
2378
2379 static int
2380 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2381 {
2382         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2383         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2384
2385         /* Disable ITR */
2386         if (hw->mac.type == fm10k_mac_pf)
2387                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2388                         FM10K_ITR_MASK_SET);
2389         else
2390                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2391                         FM10K_ITR_MASK_SET);
2392         return 0;
2393 }
2394
2395 static int
2396 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2397 {
2398         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2399         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2400         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2401         uint32_t intr_vector, vec;
2402         uint16_t queue_id;
2403         int result = 0;
2404
2405         /* fm10k needs one separate interrupt for mailbox,
2406          * so only drivers which support multiple interrupt vectors
2407          * e.g. vfio-pci can work for fm10k interrupt mode
2408          */
2409         if (!rte_intr_cap_multiple(intr_handle) ||
2410                         dev->data->dev_conf.intr_conf.rxq == 0)
2411                 return result;
2412
2413         intr_vector = dev->data->nb_rx_queues;
2414
2415         /* disable interrupt first */
2416         rte_intr_disable(intr_handle);
2417         if (hw->mac.type == fm10k_mac_pf)
2418                 fm10k_dev_disable_intr_pf(dev);
2419         else
2420                 fm10k_dev_disable_intr_vf(dev);
2421
2422         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2423                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2424                 result = -EIO;
2425         }
2426
2427         if (rte_intr_dp_is_en(intr_handle) && !result) {
2428                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2429                         dev->data->nb_rx_queues * sizeof(int), 0);
2430                 if (intr_handle->intr_vec) {
2431                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2432                                         queue_id < dev->data->nb_rx_queues;
2433                                         queue_id++) {
2434                                 intr_handle->intr_vec[queue_id] = vec;
2435                                 if (vec < intr_handle->nb_efd - 1
2436                                                 + FM10K_RX_VEC_START)
2437                                         vec++;
2438                         }
2439                 } else {
2440                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2441                                 " intr_vec", dev->data->nb_rx_queues);
2442                         rte_intr_efd_disable(intr_handle);
2443                         result = -ENOMEM;
2444                 }
2445         }
2446
2447         if (hw->mac.type == fm10k_mac_pf)
2448                 fm10k_dev_enable_intr_pf(dev);
2449         else
2450                 fm10k_dev_enable_intr_vf(dev);
2451         rte_intr_enable(intr_handle);
2452         hw->mac.ops.update_int_moderator(hw);
2453         return result;
2454 }
2455
2456 static int
2457 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2458 {
2459         struct fm10k_fault fault;
2460         int err;
2461         const char *estr = "Unknown error";
2462
2463         /* Process PCA fault */
2464         if (eicr & FM10K_EICR_PCA_FAULT) {
2465                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2466                 if (err)
2467                         goto error;
2468                 switch (fault.type) {
2469                 case PCA_NO_FAULT:
2470                         estr = "PCA_NO_FAULT"; break;
2471                 case PCA_UNMAPPED_ADDR:
2472                         estr = "PCA_UNMAPPED_ADDR"; break;
2473                 case PCA_BAD_QACCESS_PF:
2474                         estr = "PCA_BAD_QACCESS_PF"; break;
2475                 case PCA_BAD_QACCESS_VF:
2476                         estr = "PCA_BAD_QACCESS_VF"; break;
2477                 case PCA_MALICIOUS_REQ:
2478                         estr = "PCA_MALICIOUS_REQ"; break;
2479                 case PCA_POISONED_TLP:
2480                         estr = "PCA_POISONED_TLP"; break;
2481                 case PCA_TLP_ABORT:
2482                         estr = "PCA_TLP_ABORT"; break;
2483                 default:
2484                         goto error;
2485                 }
2486                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2487                         estr, fault.func ? "VF" : "PF", fault.func,
2488                         fault.address, fault.specinfo);
2489         }
2490
2491         /* Process THI fault */
2492         if (eicr & FM10K_EICR_THI_FAULT) {
2493                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2494                 if (err)
2495                         goto error;
2496                 switch (fault.type) {
2497                 case THI_NO_FAULT:
2498                         estr = "THI_NO_FAULT"; break;
2499                 case THI_MAL_DIS_Q_FAULT:
2500                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2501                 default:
2502                         goto error;
2503                 }
2504                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2505                         estr, fault.func ? "VF" : "PF", fault.func,
2506                         fault.address, fault.specinfo);
2507         }
2508
2509         /* Process FUM fault */
2510         if (eicr & FM10K_EICR_FUM_FAULT) {
2511                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2512                 if (err)
2513                         goto error;
2514                 switch (fault.type) {
2515                 case FUM_NO_FAULT:
2516                         estr = "FUM_NO_FAULT"; break;
2517                 case FUM_UNMAPPED_ADDR:
2518                         estr = "FUM_UNMAPPED_ADDR"; break;
2519                 case FUM_POISONED_TLP:
2520                         estr = "FUM_POISONED_TLP"; break;
2521                 case FUM_BAD_VF_QACCESS:
2522                         estr = "FUM_BAD_VF_QACCESS"; break;
2523                 case FUM_ADD_DECODE_ERR:
2524                         estr = "FUM_ADD_DECODE_ERR"; break;
2525                 case FUM_RO_ERROR:
2526                         estr = "FUM_RO_ERROR"; break;
2527                 case FUM_QPRC_CRC_ERROR:
2528                         estr = "FUM_QPRC_CRC_ERROR"; break;
2529                 case FUM_CSR_TIMEOUT:
2530                         estr = "FUM_CSR_TIMEOUT"; break;
2531                 case FUM_INVALID_TYPE:
2532                         estr = "FUM_INVALID_TYPE"; break;
2533                 case FUM_INVALID_LENGTH:
2534                         estr = "FUM_INVALID_LENGTH"; break;
2535                 case FUM_INVALID_BE:
2536                         estr = "FUM_INVALID_BE"; break;
2537                 case FUM_INVALID_ALIGN:
2538                         estr = "FUM_INVALID_ALIGN"; break;
2539                 default:
2540                         goto error;
2541                 }
2542                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2543                         estr, fault.func ? "VF" : "PF", fault.func,
2544                         fault.address, fault.specinfo);
2545         }
2546
2547         return 0;
2548 error:
2549         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2550         return err;
2551 }
2552
2553 /**
2554  * PF interrupt handler triggered by NIC for handling specific interrupt.
2555  *
2556  * @param handle
2557  *  Pointer to interrupt handle.
2558  * @param param
2559  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2560  *
2561  * @return
2562  *  void
2563  */
2564 static void
2565 fm10k_dev_interrupt_handler_pf(void *param)
2566 {
2567         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2568         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2569         uint32_t cause, status;
2570         struct fm10k_dev_info *dev_info =
2571                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2572         int status_mbx;
2573         s32 err;
2574
2575         if (hw->mac.type != fm10k_mac_pf)
2576                 return;
2577
2578         cause = FM10K_READ_REG(hw, FM10K_EICR);
2579
2580         /* Handle PCI fault cases */
2581         if (cause & FM10K_EICR_FAULT_MASK) {
2582                 PMD_INIT_LOG(ERR, "INT: find fault!");
2583                 fm10k_dev_handle_fault(hw, cause);
2584         }
2585
2586         /* Handle switch up/down */
2587         if (cause & FM10K_EICR_SWITCHNOTREADY)
2588                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2589
2590         if (cause & FM10K_EICR_SWITCHREADY) {
2591                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2592                 if (dev_info->sm_down == 1) {
2593                         fm10k_mbx_lock(hw);
2594
2595                         /* For recreating logical ports */
2596                         status_mbx = hw->mac.ops.update_lport_state(hw,
2597                                         hw->mac.dglort_map, MAX_LPORT_NUM, 1);
2598                         if (status_mbx == FM10K_SUCCESS)
2599                                 PMD_INIT_LOG(INFO,
2600                                         "INT: Recreated Logical port");
2601                         else
2602                                 PMD_INIT_LOG(INFO,
2603                                         "INT: Logical ports weren't recreated");
2604
2605                         status_mbx = hw->mac.ops.update_xcast_mode(hw,
2606                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2607                         if (status_mbx != FM10K_SUCCESS)
2608                                 PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2609
2610                         fm10k_mbx_unlock(hw);
2611
2612                         /* first clear the internal SW recording structure */
2613                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2614                                                 ETH_MQ_RX_VMDQ_FLAG))
2615                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2616                                         false);
2617
2618                         fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2619                                         MAIN_VSI_POOL_NUMBER);
2620
2621                         /*
2622                          * Add default mac address and vlan for the logical
2623                          * ports that have been created, leave to the
2624                          * application to fully recover Rx filtering.
2625                          */
2626                         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2627                                         MAIN_VSI_POOL_NUMBER);
2628
2629                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2630                                                 ETH_MQ_RX_VMDQ_FLAG))
2631                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2632                                         true);
2633
2634                         dev_info->sm_down = 0;
2635                         _rte_eth_dev_callback_process(dev,
2636                                         RTE_ETH_EVENT_INTR_LSC,
2637                                         NULL);
2638                 }
2639         }
2640
2641         /* Handle mailbox message */
2642         fm10k_mbx_lock(hw);
2643         err = hw->mbx.ops.process(hw, &hw->mbx);
2644         fm10k_mbx_unlock(hw);
2645
2646         if (err == FM10K_ERR_RESET_REQUESTED) {
2647                 PMD_INIT_LOG(INFO, "INT: Switch is down");
2648                 dev_info->sm_down = 1;
2649                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2650                                 NULL);
2651         }
2652
2653         /* Handle SRAM error */
2654         if (cause & FM10K_EICR_SRAMERROR) {
2655                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2656
2657                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2658                 /* Write to clear pending bits */
2659                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2660
2661                 /* Todo: print out error message after shared code  updates */
2662         }
2663
2664         /* Clear these 3 events if having any */
2665         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2666                  FM10K_EICR_SWITCHREADY;
2667         if (cause)
2668                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2669
2670         /* Re-enable interrupt from device side */
2671         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2672                                         FM10K_ITR_MASK_CLEAR);
2673         /* Re-enable interrupt from host side */
2674         rte_intr_enable(dev->intr_handle);
2675 }
2676
2677 /**
2678  * VF interrupt handler triggered by NIC for handling specific interrupt.
2679  *
2680  * @param handle
2681  *  Pointer to interrupt handle.
2682  * @param param
2683  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2684  *
2685  * @return
2686  *  void
2687  */
2688 static void
2689 fm10k_dev_interrupt_handler_vf(void *param)
2690 {
2691         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2692         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2693         struct fm10k_mbx_info *mbx = &hw->mbx;
2694         struct fm10k_dev_info *dev_info =
2695                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2696         const enum fm10k_mbx_state state = mbx->state;
2697         int status_mbx;
2698
2699         if (hw->mac.type != fm10k_mac_vf)
2700                 return;
2701
2702         /* Handle mailbox message if lock is acquired */
2703         fm10k_mbx_lock(hw);
2704         hw->mbx.ops.process(hw, &hw->mbx);
2705         fm10k_mbx_unlock(hw);
2706
2707         if (state == FM10K_STATE_OPEN && mbx->state == FM10K_STATE_CONNECT) {
2708                 PMD_INIT_LOG(INFO, "INT: Switch has gone down");
2709
2710                 fm10k_mbx_lock(hw);
2711                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2712                                 MAX_LPORT_NUM, 1);
2713                 fm10k_mbx_unlock(hw);
2714
2715                 /* Setting reset flag */
2716                 dev_info->sm_down = 1;
2717                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2718                                 NULL);
2719         }
2720
2721         if (dev_info->sm_down == 1 &&
2722                         hw->mac.dglort_map == FM10K_DGLORTMAP_ZERO) {
2723                 PMD_INIT_LOG(INFO, "INT: Switch has gone up");
2724                 fm10k_mbx_lock(hw);
2725                 status_mbx = hw->mac.ops.update_xcast_mode(hw,
2726                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2727                 if (status_mbx != FM10K_SUCCESS)
2728                         PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2729                 fm10k_mbx_unlock(hw);
2730
2731                 /* first clear the internal SW recording structure */
2732                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, false);
2733                 fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2734                                 MAIN_VSI_POOL_NUMBER);
2735
2736                 /*
2737                  * Add default mac address and vlan for the logical ports that
2738                  * have been created, leave to the application to fully recover
2739                  * Rx filtering.
2740                  */
2741                 fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2742                                 MAIN_VSI_POOL_NUMBER);
2743                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
2744
2745                 dev_info->sm_down = 0;
2746                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2747                                 NULL);
2748         }
2749
2750         /* Re-enable interrupt from device side */
2751         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2752                                         FM10K_ITR_MASK_CLEAR);
2753         /* Re-enable interrupt from host side */
2754         rte_intr_enable(dev->intr_handle);
2755 }
2756
2757 /* Mailbox message handler in VF */
2758 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2759         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2760         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2761         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2762         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2763 };
2764
2765 static int
2766 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2767 {
2768         int err = 0;
2769
2770         /* Initialize mailbox lock */
2771         fm10k_mbx_initlock(hw);
2772
2773         /* Replace default message handler with new ones */
2774         if (hw->mac.type == fm10k_mac_vf)
2775                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2776
2777         if (err) {
2778                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2779                                 err);
2780                 return err;
2781         }
2782         /* Connect to SM for PF device or PF for VF device */
2783         return hw->mbx.ops.connect(hw, &hw->mbx);
2784 }
2785
2786 static void
2787 fm10k_close_mbx_service(struct fm10k_hw *hw)
2788 {
2789         /* Disconnect from SM for PF device or PF for VF device */
2790         hw->mbx.ops.disconnect(hw, &hw->mbx);
2791 }
2792
2793 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2794         .dev_configure          = fm10k_dev_configure,
2795         .dev_start              = fm10k_dev_start,
2796         .dev_stop               = fm10k_dev_stop,
2797         .dev_close              = fm10k_dev_close,
2798         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2799         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2800         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2801         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2802         .stats_get              = fm10k_stats_get,
2803         .xstats_get             = fm10k_xstats_get,
2804         .xstats_get_names       = fm10k_xstats_get_names,
2805         .stats_reset            = fm10k_stats_reset,
2806         .xstats_reset           = fm10k_stats_reset,
2807         .link_update            = fm10k_link_update,
2808         .dev_infos_get          = fm10k_dev_infos_get,
2809         .dev_supported_ptypes_get = fm10k_dev_supported_ptypes_get,
2810         .vlan_filter_set        = fm10k_vlan_filter_set,
2811         .vlan_offload_set       = fm10k_vlan_offload_set,
2812         .mac_addr_add           = fm10k_macaddr_add,
2813         .mac_addr_remove        = fm10k_macaddr_remove,
2814         .rx_queue_start         = fm10k_dev_rx_queue_start,
2815         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2816         .tx_queue_start         = fm10k_dev_tx_queue_start,
2817         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2818         .rx_queue_setup         = fm10k_rx_queue_setup,
2819         .rx_queue_release       = fm10k_rx_queue_release,
2820         .tx_queue_setup         = fm10k_tx_queue_setup,
2821         .tx_queue_release       = fm10k_tx_queue_release,
2822         .rx_descriptor_done     = fm10k_dev_rx_descriptor_done,
2823         .rx_descriptor_status = fm10k_dev_rx_descriptor_status,
2824         .tx_descriptor_status = fm10k_dev_tx_descriptor_status,
2825         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2826         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2827         .reta_update            = fm10k_reta_update,
2828         .reta_query             = fm10k_reta_query,
2829         .rss_hash_update        = fm10k_rss_hash_update,
2830         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2831 };
2832
2833 static int ftag_check_handler(__rte_unused const char *key,
2834                 const char *value, __rte_unused void *opaque)
2835 {
2836         if (strcmp(value, "1"))
2837                 return -1;
2838
2839         return 0;
2840 }
2841
2842 static int
2843 fm10k_check_ftag(struct rte_devargs *devargs)
2844 {
2845         struct rte_kvargs *kvlist;
2846         const char *ftag_key = "enable_ftag";
2847
2848         if (devargs == NULL)
2849                 return 0;
2850
2851         kvlist = rte_kvargs_parse(devargs->args, NULL);
2852         if (kvlist == NULL)
2853                 return 0;
2854
2855         if (!rte_kvargs_count(kvlist, ftag_key)) {
2856                 rte_kvargs_free(kvlist);
2857                 return 0;
2858         }
2859         /* FTAG is enabled when there's key-value pair: enable_ftag=1 */
2860         if (rte_kvargs_process(kvlist, ftag_key,
2861                                 ftag_check_handler, NULL) < 0) {
2862                 rte_kvargs_free(kvlist);
2863                 return 0;
2864         }
2865         rte_kvargs_free(kvlist);
2866
2867         return 1;
2868 }
2869
2870 static uint16_t
2871 fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
2872                     uint16_t nb_pkts)
2873 {
2874         uint16_t nb_tx = 0;
2875         struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue;
2876
2877         while (nb_pkts) {
2878                 uint16_t ret, num;
2879
2880                 num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh);
2881                 ret = fm10k_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
2882                                                  num);
2883                 nb_tx += ret;
2884                 nb_pkts -= ret;
2885                 if (ret < num)
2886                         break;
2887         }
2888
2889         return nb_tx;
2890 }
2891
2892 static void __attribute__((cold))
2893 fm10k_set_tx_function(struct rte_eth_dev *dev)
2894 {
2895         struct fm10k_tx_queue *txq;
2896         int i;
2897         int use_sse = 1;
2898         uint16_t tx_ftag_en = 0;
2899
2900         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2901                 /* primary process has set the ftag flag and offloads */
2902                 txq = dev->data->tx_queues[0];
2903                 if (fm10k_tx_vec_condition_check(txq)) {
2904                         dev->tx_pkt_burst = fm10k_xmit_pkts;
2905                         dev->tx_pkt_prepare = fm10k_prep_pkts;
2906                         PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2907                 } else {
2908                         PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2909                         dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2910                         dev->tx_pkt_prepare = NULL;
2911                 }
2912                 return;
2913         }
2914
2915         if (fm10k_check_ftag(dev->device->devargs))
2916                 tx_ftag_en = 1;
2917
2918         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2919                 txq = dev->data->tx_queues[i];
2920                 txq->tx_ftag_en = tx_ftag_en;
2921                 /* Check if Vector Tx is satisfied */
2922                 if (fm10k_tx_vec_condition_check(txq))
2923                         use_sse = 0;
2924         }
2925
2926         if (use_sse) {
2927                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2928                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2929                         txq = dev->data->tx_queues[i];
2930                         fm10k_txq_vec_setup(txq);
2931                 }
2932                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2933                 dev->tx_pkt_prepare = NULL;
2934         } else {
2935                 dev->tx_pkt_burst = fm10k_xmit_pkts;
2936                 dev->tx_pkt_prepare = fm10k_prep_pkts;
2937                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2938         }
2939 }
2940
2941 static void __attribute__((cold))
2942 fm10k_set_rx_function(struct rte_eth_dev *dev)
2943 {
2944         struct fm10k_dev_info *dev_info =
2945                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2946         uint16_t i, rx_using_sse;
2947         uint16_t rx_ftag_en = 0;
2948
2949         if (fm10k_check_ftag(dev->device->devargs))
2950                 rx_ftag_en = 1;
2951
2952         /* In order to allow Vector Rx there are a few configuration
2953          * conditions to be met.
2954          */
2955         if (!fm10k_rx_vec_condition_check(dev) &&
2956                         dev_info->rx_vec_allowed && !rx_ftag_en) {
2957                 if (dev->data->scattered_rx)
2958                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
2959                 else
2960                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
2961         } else if (dev->data->scattered_rx)
2962                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
2963         else
2964                 dev->rx_pkt_burst = fm10k_recv_pkts;
2965
2966         rx_using_sse =
2967                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
2968                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
2969
2970         if (rx_using_sse)
2971                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
2972         else
2973                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
2974
2975         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2976                 return;
2977
2978         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2979                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
2980
2981                 rxq->rx_using_sse = rx_using_sse;
2982                 rxq->rx_ftag_en = rx_ftag_en;
2983         }
2984 }
2985
2986 static void
2987 fm10k_params_init(struct rte_eth_dev *dev)
2988 {
2989         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2990         struct fm10k_dev_info *info =
2991                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2992
2993         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
2994          * there is no way to get link status without reading BAR4.  Until this
2995          * works, assume we have maximum bandwidth.
2996          * @todo - fix bus info
2997          */
2998         hw->bus_caps.speed = fm10k_bus_speed_8000;
2999         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
3000         hw->bus_caps.payload = fm10k_bus_payload_512;
3001         hw->bus.speed = fm10k_bus_speed_8000;
3002         hw->bus.width = fm10k_bus_width_pcie_x8;
3003         hw->bus.payload = fm10k_bus_payload_256;
3004
3005         info->rx_vec_allowed = true;
3006 }
3007
3008 static int
3009 eth_fm10k_dev_init(struct rte_eth_dev *dev)
3010 {
3011         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3012         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3013         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3014         int diag, i;
3015         struct fm10k_macvlan_filter_info *macvlan;
3016
3017         PMD_INIT_FUNC_TRACE();
3018
3019         dev->dev_ops = &fm10k_eth_dev_ops;
3020         dev->rx_pkt_burst = &fm10k_recv_pkts;
3021         dev->tx_pkt_burst = &fm10k_xmit_pkts;
3022         dev->tx_pkt_prepare = &fm10k_prep_pkts;
3023
3024         /*
3025          * Primary process does the whole initialization, for secondary
3026          * processes, we just select the same Rx and Tx function as primary.
3027          */
3028         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
3029                 fm10k_set_rx_function(dev);
3030                 fm10k_set_tx_function(dev);
3031                 return 0;
3032         }
3033
3034         rte_eth_copy_pci_info(dev, pdev);
3035
3036         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
3037         memset(macvlan, 0, sizeof(*macvlan));
3038         /* Vendor and Device ID need to be set before init of shared code */
3039         memset(hw, 0, sizeof(*hw));
3040         hw->device_id = pdev->id.device_id;
3041         hw->vendor_id = pdev->id.vendor_id;
3042         hw->subsystem_device_id = pdev->id.subsystem_device_id;
3043         hw->subsystem_vendor_id = pdev->id.subsystem_vendor_id;
3044         hw->revision_id = 0;
3045         hw->hw_addr = (void *)pdev->mem_resource[0].addr;
3046         if (hw->hw_addr == NULL) {
3047                 PMD_INIT_LOG(ERR, "Bad mem resource."
3048                         " Try to blacklist unused devices.");
3049                 return -EIO;
3050         }
3051
3052         /* Store fm10k_adapter pointer */
3053         hw->back = dev->data->dev_private;
3054
3055         /* Initialize the shared code */
3056         diag = fm10k_init_shared_code(hw);
3057         if (diag != FM10K_SUCCESS) {
3058                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
3059                 return -EIO;
3060         }
3061
3062         /* Initialize parameters */
3063         fm10k_params_init(dev);
3064
3065         /* Initialize the hw */
3066         diag = fm10k_init_hw(hw);
3067         if (diag != FM10K_SUCCESS) {
3068                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
3069                 return -EIO;
3070         }
3071
3072         /* Initialize MAC address(es) */
3073         dev->data->mac_addrs = rte_zmalloc("fm10k",
3074                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
3075         if (dev->data->mac_addrs == NULL) {
3076                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
3077                 return -ENOMEM;
3078         }
3079
3080         diag = fm10k_read_mac_addr(hw);
3081
3082         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
3083                         &dev->data->mac_addrs[0]);
3084
3085         if (diag != FM10K_SUCCESS ||
3086                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
3087
3088                 /* Generate a random addr */
3089                 eth_random_addr(hw->mac.addr);
3090                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
3091                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
3092                 &dev->data->mac_addrs[0]);
3093         }
3094
3095         /* Reset the hw statistics */
3096         fm10k_stats_reset(dev);
3097
3098         /* Reset the hw */
3099         diag = fm10k_reset_hw(hw);
3100         if (diag != FM10K_SUCCESS) {
3101                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
3102                 return -EIO;
3103         }
3104
3105         /* Setup mailbox service */
3106         diag = fm10k_setup_mbx_service(hw);
3107         if (diag != FM10K_SUCCESS) {
3108                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
3109                 return -EIO;
3110         }
3111
3112         /*PF/VF has different interrupt handling mechanism */
3113         if (hw->mac.type == fm10k_mac_pf) {
3114                 /* register callback func to eal lib */
3115                 rte_intr_callback_register(intr_handle,
3116                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3117
3118                 /* enable MISC interrupt */
3119                 fm10k_dev_enable_intr_pf(dev);
3120         } else { /* VF */
3121                 rte_intr_callback_register(intr_handle,
3122                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3123
3124                 fm10k_dev_enable_intr_vf(dev);
3125         }
3126
3127         /* Enable intr after callback registered */
3128         rte_intr_enable(intr_handle);
3129
3130         hw->mac.ops.update_int_moderator(hw);
3131
3132         /* Make sure Switch Manager is ready before going forward. */
3133         if (hw->mac.type == fm10k_mac_pf) {
3134                 int switch_ready = 0;
3135
3136                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3137                         fm10k_mbx_lock(hw);
3138                         hw->mac.ops.get_host_state(hw, &switch_ready);
3139                         fm10k_mbx_unlock(hw);
3140                         if (switch_ready)
3141                                 break;
3142                         /* Delay some time to acquire async LPORT_MAP info. */
3143                         rte_delay_us(WAIT_SWITCH_MSG_US);
3144                 }
3145
3146                 if (switch_ready == 0) {
3147                         PMD_INIT_LOG(ERR, "switch is not ready");
3148                         return -1;
3149                 }
3150         }
3151
3152         /*
3153          * Below function will trigger operations on mailbox, acquire lock to
3154          * avoid race condition from interrupt handler. Operations on mailbox
3155          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
3156          * will handle and generate an interrupt to our side. Then,  FIFO in
3157          * mailbox will be touched.
3158          */
3159         fm10k_mbx_lock(hw);
3160         /* Enable port first */
3161         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
3162                                         MAX_LPORT_NUM, 1);
3163
3164         /* Set unicast mode by default. App can change to other mode in other
3165          * API func.
3166          */
3167         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
3168                                         FM10K_XCAST_MODE_NONE);
3169
3170         fm10k_mbx_unlock(hw);
3171
3172         /* Make sure default VID is ready before going forward. */
3173         if (hw->mac.type == fm10k_mac_pf) {
3174                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3175                         if (hw->mac.default_vid)
3176                                 break;
3177                         /* Delay some time to acquire async port VLAN info. */
3178                         rte_delay_us(WAIT_SWITCH_MSG_US);
3179                 }
3180
3181                 if (!hw->mac.default_vid) {
3182                         PMD_INIT_LOG(ERR, "default VID is not ready");
3183                         return -1;
3184                 }
3185         }
3186
3187         /* Add default mac address */
3188         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
3189                 MAIN_VSI_POOL_NUMBER);
3190
3191         return 0;
3192 }
3193
3194 static int
3195 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
3196 {
3197         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3198         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3199         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3200         PMD_INIT_FUNC_TRACE();
3201
3202         /* only uninitialize in the primary process */
3203         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3204                 return 0;
3205
3206         /* safe to close dev here */
3207         fm10k_dev_close(dev);
3208
3209         dev->dev_ops = NULL;
3210         dev->rx_pkt_burst = NULL;
3211         dev->tx_pkt_burst = NULL;
3212
3213         /* disable uio/vfio intr */
3214         rte_intr_disable(intr_handle);
3215
3216         /*PF/VF has different interrupt handling mechanism */
3217         if (hw->mac.type == fm10k_mac_pf) {
3218                 /* disable interrupt */
3219                 fm10k_dev_disable_intr_pf(dev);
3220
3221                 /* unregister callback func to eal lib */
3222                 rte_intr_callback_unregister(intr_handle,
3223                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3224         } else {
3225                 /* disable interrupt */
3226                 fm10k_dev_disable_intr_vf(dev);
3227
3228                 rte_intr_callback_unregister(intr_handle,
3229                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3230         }
3231
3232         return 0;
3233 }
3234
3235 static int eth_fm10k_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
3236         struct rte_pci_device *pci_dev)
3237 {
3238         return rte_eth_dev_pci_generic_probe(pci_dev,
3239                 sizeof(struct fm10k_adapter), eth_fm10k_dev_init);
3240 }
3241
3242 static int eth_fm10k_pci_remove(struct rte_pci_device *pci_dev)
3243 {
3244         return rte_eth_dev_pci_generic_remove(pci_dev, eth_fm10k_dev_uninit);
3245 }
3246
3247 /*
3248  * The set of PCI devices this driver supports. This driver will enable both PF
3249  * and SRIOV-VF devices.
3250  */
3251 static const struct rte_pci_id pci_id_fm10k_map[] = {
3252         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_PF) },
3253         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_SDI_FM10420_QDA2) },
3254         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_VF) },
3255         { .vendor_id = 0, /* sentinel */ },
3256 };
3257
3258 static struct rte_pci_driver rte_pmd_fm10k = {
3259         .id_table = pci_id_fm10k_map,
3260         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
3261                      RTE_PCI_DRV_IOVA_AS_VA,
3262         .probe = eth_fm10k_pci_probe,
3263         .remove = eth_fm10k_pci_remove,
3264 };
3265
3266 RTE_PMD_REGISTER_PCI(net_fm10k, rte_pmd_fm10k);
3267 RTE_PMD_REGISTER_PCI_TABLE(net_fm10k, pci_id_fm10k_map);
3268 RTE_PMD_REGISTER_KMOD_DEP(net_fm10k, "* igb_uio | uio_pci_generic | vfio-pci");
3269
3270 RTE_INIT(fm10k_init_log)
3271 {
3272         fm10k_logtype_init = rte_log_register("pmd.net.fm10k.init");
3273         if (fm10k_logtype_init >= 0)
3274                 rte_log_set_level(fm10k_logtype_init, RTE_LOG_NOTICE);
3275         fm10k_logtype_driver = rte_log_register("pmd.net.fm10k.driver");
3276         if (fm10k_logtype_driver >= 0)
3277                 rte_log_set_level(fm10k_logtype_driver, RTE_LOG_NOTICE);
3278 }