drivers/net/fm10k/fm10k_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2013-2016 Intel Corporation
   3  */
   4
   5 #include <rte_ethdev_driver.h>
   6 #include <rte_ethdev_pci.h>
   7 #include <rte_malloc.h>
   8 #include <rte_memzone.h>
   9 #include <rte_string_fns.h>
  10 #include <rte_dev.h>
  11 #include <rte_spinlock.h>
  12 #include <rte_kvargs.h>
  13
  14 #include "fm10k.h"
  15 #include "base/fm10k_api.h"
  16
  17 /* Default delay to acquire mailbox lock */
  18 #define FM10K_MBXLOCK_DELAY_US 20
  19 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  20
  21 #define MAIN_VSI_POOL_NUMBER 0
  22
  23 /* Max try times to acquire switch status */
  24 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  25 /* Wait interval to get switch status */
  26 #define WAIT_SWITCH_MSG_US    100000
  27 /* A period of quiescence for switch */
  28 #define FM10K_SWITCH_QUIESCE_US 100000
  29 /* Number of chars per uint32 type */
  30 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  31 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  32
  33 /* default 1:1 map from queue ID to interrupt vector ID */
  34 #define Q2V(pci_dev, queue_id) ((pci_dev)->intr_handle.intr_vec[queue_id])
  35
  36 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  37 #define MAX_LPORT_NUM    128
  38 #define GLORT_FD_Q_BASE  0x40
  39 #define GLORT_PF_MASK    0xFFC0
  40 #define GLORT_FD_MASK    GLORT_PF_MASK
  41 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  42
  43 int fm10k_logtype_init;
  44 int fm10k_logtype_driver;
  45
  46 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  47 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  48 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  49 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  50 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  51 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  52 static int
  53 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  54 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  55         const u8 *mac, bool add, uint32_t pool);
  56 static void fm10k_tx_queue_release(void *queue);
  57 static void fm10k_rx_queue_release(void *queue);
  58 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  59 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  60 static int fm10k_check_ftag(struct rte_devargs *devargs);
  61 static int fm10k_link_update(struct rte_eth_dev *dev, int wait_to_complete);
  62
  63 static void fm10k_dev_infos_get(struct rte_eth_dev *dev,
  64                                 struct rte_eth_dev_info *dev_info);
  65 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev);
  66 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev);
  67 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev);
  68 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev);
  69
  70 struct fm10k_xstats_name_off {
  71         char name[RTE_ETH_XSTATS_NAME_SIZE];
  72         unsigned offset;
  73 };
  74
  75 struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  76         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  77         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  78         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  79         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  80         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  81         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
  82         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
  83         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
  84                 nodesc_drop)},
  85 };
  86
  87 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
  88                 sizeof(fm10k_hw_stats_strings[0]))
  89
  90 struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
  91         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
  92         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
  93         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
  94 };
  95
  96 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
  97                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
  98
  99 struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
 100         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 101         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 102 };
 103
 104 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 105                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 106
 107 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 108                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 109 static int
 110 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 111
 112 static void
 113 fm10k_mbx_initlock(struct fm10k_hw *hw)
 114 {
 115         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 116 }
 117
 118 static void
 119 fm10k_mbx_lock(struct fm10k_hw *hw)
 120 {
 121         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 122                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 123 }
 124
 125 static void
 126 fm10k_mbx_unlock(struct fm10k_hw *hw)
 127 {
 128         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 129 }
 130
 131 /* Stubs needed for linkage when vPMD is disabled */
 132 int __attribute__((weak))
 133 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 134 {
 135         return -1;
 136 }
 137
 138 uint16_t __attribute__((weak))
 139 fm10k_recv_pkts_vec(
 140         __rte_unused void *rx_queue,
 141         __rte_unused struct rte_mbuf **rx_pkts,
 142         __rte_unused uint16_t nb_pkts)
 143 {
 144         return 0;
 145 }
 146
 147 uint16_t __attribute__((weak))
 148 fm10k_recv_scattered_pkts_vec(
 149                 __rte_unused void *rx_queue,
 150                 __rte_unused struct rte_mbuf **rx_pkts,
 151                 __rte_unused uint16_t nb_pkts)
 152 {
 153         return 0;
 154 }
 155
 156 int __attribute__((weak))
 157 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 158
 159 {
 160         return -1;
 161 }
 162
 163 void __attribute__((weak))
 164 fm10k_rx_queue_release_mbufs_vec(
 165                 __rte_unused struct fm10k_rx_queue *rxq)
 166 {
 167         return;
 168 }
 169
 170 void __attribute__((weak))
 171 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 172 {
 173         return;
 174 }
 175
 176 int __attribute__((weak))
 177 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 178 {
 179         return -1;
 180 }
 181
 182 uint16_t __attribute__((weak))
 183 fm10k_xmit_fixed_burst_vec(__rte_unused void *tx_queue,
 184                            __rte_unused struct rte_mbuf **tx_pkts,
 185                            __rte_unused uint16_t nb_pkts)
 186 {
 187         return 0;
 188 }
 189
 190 /*
 191  * reset queue to initial state, allocate software buffers used when starting
 192  * device.
 193  * return 0 on success
 194  * return -ENOMEM if buffers cannot be allocated
 195  * return -EINVAL if buffers do not satisfy alignment condition
 196  */
 197 static inline int
 198 rx_queue_reset(struct fm10k_rx_queue *q)
 199 {
 200         static const union fm10k_rx_desc zero = {{0} };
 201         uint64_t dma_addr;
 202         int i, diag;
 203         PMD_INIT_FUNC_TRACE();
 204
 205         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 206         if (diag != 0)
 207                 return -ENOMEM;
 208
 209         for (i = 0; i < q->nb_desc; ++i) {
 210                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 211                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 212                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 213                                                 q->nb_desc);
 214                         return -EINVAL;
 215                 }
 216                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 217                 q->hw_ring[i].q.pkt_addr = dma_addr;
 218                 q->hw_ring[i].q.hdr_addr = dma_addr;
 219         }
 220
 221         /* initialize extra software ring entries. Space for these extra
 222          * entries is always allocated.
 223          */
 224         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 225         for (i = 0; i < q->nb_fake_desc; ++i) {
 226                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 227                 q->hw_ring[q->nb_desc + i] = zero;
 228         }
 229
 230         q->next_dd = 0;
 231         q->next_alloc = 0;
 232         q->next_trigger = q->alloc_thresh - 1;
 233         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 234         q->rxrearm_start = 0;
 235         q->rxrearm_nb = 0;
 236
 237         return 0;
 238 }
 239
 240 /*
 241  * clean queue, descriptor rings, free software buffers used when stopping
 242  * device.
 243  */
 244 static inline void
 245 rx_queue_clean(struct fm10k_rx_queue *q)
 246 {
 247         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 248         uint32_t i;
 249         PMD_INIT_FUNC_TRACE();
 250
 251         /* zero descriptor rings */
 252         for (i = 0; i < q->nb_desc; ++i)
 253                 q->hw_ring[i] = zero;
 254
 255         /* zero faked descriptors */
 256         for (i = 0; i < q->nb_fake_desc; ++i)
 257                 q->hw_ring[q->nb_desc + i] = zero;
 258
 259         /* vPMD driver has a different way of releasing mbufs. */
 260         if (q->rx_using_sse) {
 261                 fm10k_rx_queue_release_mbufs_vec(q);
 262                 return;
 263         }
 264
 265         /* free software buffers */
 266         for (i = 0; i < q->nb_desc; ++i) {
 267                 if (q->sw_ring[i]) {
 268                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 269                         q->sw_ring[i] = NULL;
 270                 }
 271         }
 272 }
 273
 274 /*
 275  * free all queue memory used when releasing the queue (i.e. configure)
 276  */
 277 static inline void
 278 rx_queue_free(struct fm10k_rx_queue *q)
 279 {
 280         PMD_INIT_FUNC_TRACE();
 281         if (q) {
 282                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 283                 rx_queue_clean(q);
 284                 if (q->sw_ring) {
 285                         rte_free(q->sw_ring);
 286                         q->sw_ring = NULL;
 287                 }
 288                 rte_free(q);
 289                 q = NULL;
 290         }
 291 }
 292
 293 /*
 294  * disable RX queue, wait unitl HW finished necessary flush operation
 295  */
 296 static inline int
 297 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 298 {
 299         uint32_t reg, i;
 300
 301         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 302         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 303                         reg & ~FM10K_RXQCTL_ENABLE);
 304
 305         /* Wait 100us at most */
 306         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 307                 rte_delay_us(1);
 308                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 309                 if (!(reg & FM10K_RXQCTL_ENABLE))
 310                         break;
 311         }
 312
 313         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 314                 return -1;
 315
 316         return 0;
 317 }
 318
 319 /*
 320  * reset queue to initial state, allocate software buffers used when starting
 321  * device
 322  */
 323 static inline void
 324 tx_queue_reset(struct fm10k_tx_queue *q)
 325 {
 326         PMD_INIT_FUNC_TRACE();
 327         q->last_free = 0;
 328         q->next_free = 0;
 329         q->nb_used = 0;
 330         q->nb_free = q->nb_desc - 1;
 331         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 332         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 333 }
 334
 335 /*
 336  * clean queue, descriptor rings, free software buffers used when stopping
 337  * device
 338  */
 339 static inline void
 340 tx_queue_clean(struct fm10k_tx_queue *q)
 341 {
 342         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 343         uint32_t i;
 344         PMD_INIT_FUNC_TRACE();
 345
 346         /* zero descriptor rings */
 347         for (i = 0; i < q->nb_desc; ++i)
 348                 q->hw_ring[i] = zero;
 349
 350         /* free software buffers */
 351         for (i = 0; i < q->nb_desc; ++i) {
 352                 if (q->sw_ring[i]) {
 353                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 354                         q->sw_ring[i] = NULL;
 355                 }
 356         }
 357 }
 358
 359 /*
 360  * free all queue memory used when releasing the queue (i.e. configure)
 361  */
 362 static inline void
 363 tx_queue_free(struct fm10k_tx_queue *q)
 364 {
 365         PMD_INIT_FUNC_TRACE();
 366         if (q) {
 367                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 368                 tx_queue_clean(q);
 369                 if (q->rs_tracker.list) {
 370                         rte_free(q->rs_tracker.list);
 371                         q->rs_tracker.list = NULL;
 372                 }
 373                 if (q->sw_ring) {
 374                         rte_free(q->sw_ring);
 375                         q->sw_ring = NULL;
 376                 }
 377                 rte_free(q);
 378                 q = NULL;
 379         }
 380 }
 381
 382 /*
 383  * disable TX queue, wait unitl HW finished necessary flush operation
 384  */
 385 static inline int
 386 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 387 {
 388         uint32_t reg, i;
 389
 390         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 391         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 392                         reg & ~FM10K_TXDCTL_ENABLE);
 393
 394         /* Wait 100us at most */
 395         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 396                 rte_delay_us(1);
 397                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 398                 if (!(reg & FM10K_TXDCTL_ENABLE))
 399                         break;
 400         }
 401
 402         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 403                 return -1;
 404
 405         return 0;
 406 }
 407
 408 static int
 409 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 410 {
 411         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 412         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 413         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 414         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 415
 416         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 417
 418         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 419                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 420                 return -EINVAL;
 421         }
 422
 423         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 424                 return 0;
 425
 426         if (hw->mac.type == fm10k_mac_vf) {
 427                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 428                 return -EINVAL;
 429         }
 430
 431         /* Check VMDQ queue pool number */
 432         if (vmdq_conf->nb_queue_pools >
 433                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 434                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 435                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 436                         vmdq_conf->nb_queue_pools);
 437                 return -EINVAL;
 438         }
 439
 440         return 0;
 441 }
 442
 443 static const struct fm10k_txq_ops def_txq_ops = {
 444         .reset = tx_queue_reset,
 445 };
 446
 447 static int
 448 fm10k_dev_configure(struct rte_eth_dev *dev)
 449 {
 450         int ret;
 451
 452         PMD_INIT_FUNC_TRACE();
 453
 454         /* KEEP_CRC offload flag is not supported by PMD
 455          * can remove the below block when DEV_RX_OFFLOAD_CRC_STRIP removed
 456          */
 457         if (rte_eth_dev_must_keep_crc(dev->data->dev_conf.rxmode.offloads))
 458                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 459
 460         /* multipe queue mode checking */
 461         ret  = fm10k_check_mq_mode(dev);
 462         if (ret != 0) {
 463                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 464                             ret);
 465                 return ret;
 466         }
 467
 468         dev->data->scattered_rx = 0;
 469
 470         return 0;
 471 }
 472
 473 /* fls = find last set bit = 32 minus the number of leading zeros */
 474 #ifndef fls
 475 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 476 #endif
 477
 478 static void
 479 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 480 {
 481         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 482         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 483         uint32_t i;
 484
 485         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 486
 487         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 488                 if (!vmdq_conf->pool_map[i].pools)
 489                         continue;
 490                 fm10k_mbx_lock(hw);
 491                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 492                 fm10k_mbx_unlock(hw);
 493         }
 494 }
 495
 496 static void
 497 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 498 {
 499         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 500
 501         /* Add default mac address */
 502         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 503                 MAIN_VSI_POOL_NUMBER);
 504 }
 505
 506 static void
 507 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 508 {
 509         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 510         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 511         uint32_t mrqc, *key, i, reta, j;
 512         uint64_t hf;
 513
 514 #define RSS_KEY_SIZE 40
 515         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 516                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 517                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 518                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 519                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 520                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 521         };
 522
 523         if (dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 524                 dev_conf->rx_adv_conf.rss_conf.rss_hf == 0) {
 525                 FM10K_WRITE_REG(hw, FM10K_MRQC(0), 0);
 526                 return;
 527         }
 528
 529         /* random key is rss_intel_key (default) or user provided (rss_key) */
 530         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 531                 key = (uint32_t *)rss_intel_key;
 532         else
 533                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 534
 535         /* Now fill our hash function seeds, 4 bytes at a time */
 536         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 537                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 538
 539         /*
 540          * Fill in redirection table
 541          * The byte-swap is needed because NIC registers are in
 542          * little-endian order.
 543          */
 544         reta = 0;
 545         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 546                 if (j == dev->data->nb_rx_queues)
 547                         j = 0;
 548                 reta = (reta << CHAR_BIT) | j;
 549                 if ((i & 3) == 3)
 550                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 551                                         rte_bswap32(reta));
 552         }
 553
 554         /*
 555          * Generate RSS hash based on packet types, TCP/UDP
 556          * port numbers and/or IPv4/v6 src and dst addresses
 557          */
 558         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 559         mrqc = 0;
 560         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 561         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 562         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 563         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 564         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 565         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 566         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 567         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 568         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 569
 570         if (mrqc == 0) {
 571                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 572                         "supported", hf);
 573                 return;
 574         }
 575
 576         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 577 }
 578
 579 static void
 580 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 581 {
 582         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 583         uint32_t i;
 584
 585         for (i = 0; i < nb_lport_new; i++) {
 586                 /* Set unicast mode by default. App can change
 587                  * to other mode in other API func.
 588                  */
 589                 fm10k_mbx_lock(hw);
 590                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 591                         FM10K_XCAST_MODE_NONE);
 592                 fm10k_mbx_unlock(hw);
 593         }
 594 }
 595
 596 static void
 597 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 598 {
 599         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 600         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 601         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 602         struct fm10k_macvlan_filter_info *macvlan;
 603         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 604         uint16_t nb_lport_new;
 605
 606         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 607         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 608
 609         fm10k_dev_rss_configure(dev);
 610
 611         /* only PF supports VMDQ */
 612         if (hw->mac.type != fm10k_mac_pf)
 613                 return;
 614
 615         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 616                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 617
 618         /* no pool number change, no need to update logic port and VLAN/MAC */
 619         if (macvlan->nb_queue_pools == nb_queue_pools)
 620                 return;
 621
 622         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 623         fm10k_dev_logic_port_update(dev, nb_lport_new);
 624
 625         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 626         memset(dev->data->mac_addrs, 0,
 627                 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 628         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
 629                 &dev->data->mac_addrs[0]);
 630         memset(macvlan, 0, sizeof(*macvlan));
 631         macvlan->nb_queue_pools = nb_queue_pools;
 632
 633         if (nb_queue_pools)
 634                 fm10k_dev_vmdq_rx_configure(dev);
 635         else
 636                 fm10k_dev_pf_main_vsi_reset(dev);
 637 }
 638
 639 static int
 640 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 641 {
 642         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 643         int i, ret;
 644         struct fm10k_tx_queue *txq;
 645         uint64_t base_addr;
 646         uint32_t size;
 647
 648         /* Disable TXINT to avoid possible interrupt */
 649         for (i = 0; i < hw->mac.max_queues; i++)
 650                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 651                                 3 << FM10K_TXINT_TIMER_SHIFT);
 652
 653         /* Setup TX queue */
 654         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 655                 txq = dev->data->tx_queues[i];
 656                 base_addr = txq->hw_ring_phys_addr;
 657                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 658
 659                 /* disable queue to avoid issues while updating state */
 660                 ret = tx_queue_disable(hw, i);
 661                 if (ret) {
 662                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 663                         return -1;
 664                 }
 665                 /* Enable use of FTAG bit in TX descriptor, PFVTCTL
 666                  * register is read-only for VF.
 667                  */
 668                 if (fm10k_check_ftag(dev->device->devargs)) {
 669                         if (hw->mac.type == fm10k_mac_pf) {
 670                                 FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i),
 671                                                 FM10K_PFVTCTL_FTAG_DESC_ENABLE);
 672                                 PMD_INIT_LOG(DEBUG, "FTAG mode is enabled");
 673                         } else {
 674                                 PMD_INIT_LOG(ERR, "VF FTAG is not supported.");
 675                                 return -ENOTSUP;
 676                         }
 677                 }
 678
 679                 /* set location and size for descriptor ring */
 680                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 681                                 base_addr & UINT64_LOWER_32BITS_MASK);
 682                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 683                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 684                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 685
 686                 /* assign default SGLORT for each TX queue by PF */
 687                 if (hw->mac.type == fm10k_mac_pf)
 688                         FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map);
 689         }
 690
 691         /* set up vector or scalar TX function as appropriate */
 692         fm10k_set_tx_function(dev);
 693
 694         return 0;
 695 }
 696
 697 static int
 698 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 699 {
 700         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 701         struct fm10k_macvlan_filter_info *macvlan;
 702         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
 703         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
 704         int i, ret;
 705         struct fm10k_rx_queue *rxq;
 706         uint64_t base_addr;
 707         uint32_t size;
 708         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 709         uint32_t logic_port = hw->mac.dglort_map;
 710         uint16_t buf_size;
 711         uint16_t queue_stride = 0;
 712
 713         /* enable RXINT for interrupt mode */
 714         i = 0;
 715         if (rte_intr_dp_is_en(intr_handle)) {
 716                 for (; i < dev->data->nb_rx_queues; i++) {
 717                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(pdev, i));
 718                         if (hw->mac.type == fm10k_mac_pf)
 719                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
 720                                         FM10K_ITR_AUTOMASK |
 721                                         FM10K_ITR_MASK_CLEAR);
 722                         else
 723                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
 724                                         FM10K_ITR_AUTOMASK |
 725                                         FM10K_ITR_MASK_CLEAR);
 726                 }
 727         }
 728         /* Disable other RXINT to avoid possible interrupt */
 729         for (; i < hw->mac.max_queues; i++)
 730                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 731                         3 << FM10K_RXINT_TIMER_SHIFT);
 732
 733         /* Setup RX queues */
 734         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 735                 rxq = dev->data->rx_queues[i];
 736                 base_addr = rxq->hw_ring_phys_addr;
 737                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 738
 739                 /* disable queue to avoid issues while updating state */
 740                 ret = rx_queue_disable(hw, i);
 741                 if (ret) {
 742                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 743                         return -1;
 744                 }
 745
 746                 /* Setup the Base and Length of the Rx Descriptor Ring */
 747                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 748                                 base_addr & UINT64_LOWER_32BITS_MASK);
 749                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 750                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 751                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 752
 753                 /* Configure the Rx buffer size for one buff without split */
 754                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 755                         RTE_PKTMBUF_HEADROOM);
 756                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 757                  * reserved for this purpose, and the worst case could be 511B.
 758                  * But SRR reg assumes all buffers have the same size. In order
 759                  * to fill the gap, we'll have to consider the worst case and
 760                  * assume 512B is reserved. If we don't do so, it's possible
 761                  * for HW to overwrite data to next mbuf.
 762                  */
 763                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 764
 765                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 766                                 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
 767                                 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
 768
 769                 /* It adds dual VLAN length for supporting dual VLAN */
 770                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 771                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 772                         rxq->offloads & DEV_RX_OFFLOAD_SCATTER) {
 773                         uint32_t reg;
 774                         dev->data->scattered_rx = 1;
 775                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 776                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 777                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 778                 }
 779
 780                 /* Enable drop on empty, it's RO for VF */
 781                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 782                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 783
 784                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 785                 FM10K_WRITE_FLUSH(hw);
 786         }
 787
 788         /* Configure VMDQ/RSS if applicable */
 789         fm10k_dev_mq_rx_configure(dev);
 790
 791         /* Decide the best RX function */
 792         fm10k_set_rx_function(dev);
 793
 794         /* update RX_SGLORT for loopback suppress*/
 795         if (hw->mac.type != fm10k_mac_pf)
 796                 return 0;
 797         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 798         if (macvlan->nb_queue_pools)
 799                 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
 800         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 801                 if (i && queue_stride && !(i % queue_stride))
 802                         logic_port++;
 803                 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
 804         }
 805
 806         return 0;
 807 }
 808
 809 static int
 810 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 811 {
 812         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 813         int err;
 814         uint32_t reg;
 815         struct fm10k_rx_queue *rxq;
 816
 817         PMD_INIT_FUNC_TRACE();
 818
 819         rxq = dev->data->rx_queues[rx_queue_id];
 820         err = rx_queue_reset(rxq);
 821         if (err == -ENOMEM) {
 822                 PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 823                 return err;
 824         } else if (err == -EINVAL) {
 825                 PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 826                         " %d", err);
 827                 return err;
 828         }
 829
 830         /* Setup the HW Rx Head and Tail Descriptor Pointers
 831          * Note: this must be done AFTER the queue is enabled on real
 832          * hardware, but BEFORE the queue is enabled when using the
 833          * emulation platform. Do it in both places for now and remove
 834          * this comment and the following two register writes when the
 835          * emulation platform is no longer being used.
 836          */
 837         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 838         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 839
 840         /* Set PF ownership flag for PF devices */
 841         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 842         if (hw->mac.type == fm10k_mac_pf)
 843                 reg |= FM10K_RXQCTL_PF;
 844         reg |= FM10K_RXQCTL_ENABLE;
 845         /* enable RX queue */
 846         FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 847         FM10K_WRITE_FLUSH(hw);
 848
 849         /* Setup the HW Rx Head and Tail Descriptor Pointers
 850          * Note: this must be done AFTER the queue is enabled
 851          */
 852         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 853         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 854         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 855
 856         return 0;
 857 }
 858
 859 static int
 860 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 861 {
 862         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 863
 864         PMD_INIT_FUNC_TRACE();
 865
 866         /* Disable RX queue */
 867         rx_queue_disable(hw, rx_queue_id);
 868
 869         /* Free mbuf and clean HW ring */
 870         rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 871         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 872
 873         return 0;
 874 }
 875
 876 static int
 877 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 878 {
 879         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 880         /** @todo - this should be defined in the shared code */
 881 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 882         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 883         struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 884
 885         PMD_INIT_FUNC_TRACE();
 886
 887         q->ops->reset(q);
 888
 889         /* reset head and tail pointers */
 890         FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 891         FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 892
 893         /* enable TX queue */
 894         FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 895                                 FM10K_TXDCTL_ENABLE | txdctl);
 896         FM10K_WRITE_FLUSH(hw);
 897         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 898
 899         return 0;
 900 }
 901
 902 static int
 903 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 904 {
 905         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 906
 907         PMD_INIT_FUNC_TRACE();
 908
 909         tx_queue_disable(hw, tx_queue_id);
 910         tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 911         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 912
 913         return 0;
 914 }
 915
 916 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 917 {
 918         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 919                 != FM10K_DGLORTMAP_NONE);
 920 }
 921
 922 static void
 923 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 924 {
 925         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 926         int status;
 927
 928         PMD_INIT_FUNC_TRACE();
 929
 930         /* Return if it didn't acquire valid glort range */
 931         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 932                 return;
 933
 934         fm10k_mbx_lock(hw);
 935         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 936                                 FM10K_XCAST_MODE_PROMISC);
 937         fm10k_mbx_unlock(hw);
 938
 939         if (status != FM10K_SUCCESS)
 940                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 941 }
 942
 943 static void
 944 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 945 {
 946         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 947         uint8_t mode;
 948         int status;
 949
 950         PMD_INIT_FUNC_TRACE();
 951
 952         /* Return if it didn't acquire valid glort range */
 953         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 954                 return;
 955
 956         if (dev->data->all_multicast == 1)
 957                 mode = FM10K_XCAST_MODE_ALLMULTI;
 958         else
 959                 mode = FM10K_XCAST_MODE_NONE;
 960
 961         fm10k_mbx_lock(hw);
 962         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 963                                 mode);
 964         fm10k_mbx_unlock(hw);
 965
 966         if (status != FM10K_SUCCESS)
 967                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 968 }
 969
 970 static void
 971 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 972 {
 973         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 974         int status;
 975
 976         PMD_INIT_FUNC_TRACE();
 977
 978         /* Return if it didn't acquire valid glort range */
 979         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 980                 return;
 981
 982         /* If promiscuous mode is enabled, it doesn't make sense to enable
 983          * allmulticast and disable promiscuous since fm10k only can select
 984          * one of the modes.
 985          */
 986         if (dev->data->promiscuous) {
 987                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 988                         "needn't enable allmulticast");
 989                 return;
 990         }
 991
 992         fm10k_mbx_lock(hw);
 993         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 994                                 FM10K_XCAST_MODE_ALLMULTI);
 995         fm10k_mbx_unlock(hw);
 996
 997         if (status != FM10K_SUCCESS)
 998                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 999 }
1000
1001 static void
1002 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
1003 {
1004         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1005         int status;
1006
1007         PMD_INIT_FUNC_TRACE();
1008
1009         /* Return if it didn't acquire valid glort range */
1010         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1011                 return;
1012
1013         if (dev->data->promiscuous) {
1014                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1015                         "since promisc mode is enabled");
1016                 return;
1017         }
1018
1019         fm10k_mbx_lock(hw);
1020         /* Change mode to unicast mode */
1021         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1022                                 FM10K_XCAST_MODE_NONE);
1023         fm10k_mbx_unlock(hw);
1024
1025         if (status != FM10K_SUCCESS)
1026                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1027 }
1028
1029 static void
1030 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1031 {
1032         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1033         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1034         uint16_t nb_queue_pools;
1035         struct fm10k_macvlan_filter_info *macvlan;
1036
1037         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1038         nb_queue_pools = macvlan->nb_queue_pools;
1039         pool_len = nb_queue_pools ? fls(nb_queue_pools - 1) : 0;
1040         rss_len = fls(dev->data->nb_rx_queues - 1) - pool_len;
1041
1042         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1043         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1044         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1045                         hw->mac.dglort_map;
1046         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1047         /* Configure VMDQ/RSS DGlort Decoder */
1048         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1049
1050         /* Flow Director configurations, only queue number is valid. */
1051         dglortdec = fls(dev->data->nb_rx_queues - 1);
1052         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1053                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1054         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1055         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1056
1057         /* Invalidate all other GLORT entries */
1058         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1059                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1060                                 FM10K_DGLORTMAP_NONE);
1061 }
1062
1063 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1064 static int
1065 fm10k_dev_start(struct rte_eth_dev *dev)
1066 {
1067         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1068         int i, diag;
1069
1070         PMD_INIT_FUNC_TRACE();
1071
1072         /* stop, init, then start the hw */
1073         diag = fm10k_stop_hw(hw);
1074         if (diag != FM10K_SUCCESS) {
1075                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1076                 return -EIO;
1077         }
1078
1079         diag = fm10k_init_hw(hw);
1080         if (diag != FM10K_SUCCESS) {
1081                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1082                 return -EIO;
1083         }
1084
1085         diag = fm10k_start_hw(hw);
1086         if (diag != FM10K_SUCCESS) {
1087                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1088                 return -EIO;
1089         }
1090
1091         diag = fm10k_dev_tx_init(dev);
1092         if (diag) {
1093                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1094                 return diag;
1095         }
1096
1097         if (fm10k_dev_rxq_interrupt_setup(dev))
1098                 return -EIO;
1099
1100         diag = fm10k_dev_rx_init(dev);
1101         if (diag) {
1102                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1103                 return diag;
1104         }
1105
1106         if (hw->mac.type == fm10k_mac_pf)
1107                 fm10k_dev_dglort_map_configure(dev);
1108
1109         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1110                 struct fm10k_rx_queue *rxq;
1111                 rxq = dev->data->rx_queues[i];
1112
1113                 if (rxq->rx_deferred_start)
1114                         continue;
1115                 diag = fm10k_dev_rx_queue_start(dev, i);
1116                 if (diag != 0) {
1117                         int j;
1118                         for (j = 0; j < i; ++j)
1119                                 rx_queue_clean(dev->data->rx_queues[j]);
1120                         return diag;
1121                 }
1122         }
1123
1124         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1125                 struct fm10k_tx_queue *txq;
1126                 txq = dev->data->tx_queues[i];
1127
1128                 if (txq->tx_deferred_start)
1129                         continue;
1130                 diag = fm10k_dev_tx_queue_start(dev, i);
1131                 if (diag != 0) {
1132                         int j;
1133                         for (j = 0; j < i; ++j)
1134                                 tx_queue_clean(dev->data->tx_queues[j]);
1135                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1136                                 rx_queue_clean(dev->data->rx_queues[j]);
1137                         return diag;
1138                 }
1139         }
1140
1141         /* Update default vlan when not in VMDQ mode */
1142         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1143                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1144
1145         fm10k_link_update(dev, 0);
1146
1147         return 0;
1148 }
1149
1150 static void
1151 fm10k_dev_stop(struct rte_eth_dev *dev)
1152 {
1153         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1154         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1155         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
1156         int i;
1157
1158         PMD_INIT_FUNC_TRACE();
1159
1160         if (dev->data->tx_queues)
1161                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1162                         fm10k_dev_tx_queue_stop(dev, i);
1163
1164         if (dev->data->rx_queues)
1165                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1166                         fm10k_dev_rx_queue_stop(dev, i);
1167
1168         /* Disable datapath event */
1169         if (rte_intr_dp_is_en(intr_handle)) {
1170                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1171                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1172                                 3 << FM10K_RXINT_TIMER_SHIFT);
1173                         if (hw->mac.type == fm10k_mac_pf)
1174                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
1175                                         FM10K_ITR_MASK_SET);
1176                         else
1177                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
1178                                         FM10K_ITR_MASK_SET);
1179                 }
1180         }
1181         /* Clean datapath event and queue/vec mapping */
1182         rte_intr_efd_disable(intr_handle);
1183         rte_free(intr_handle->intr_vec);
1184         intr_handle->intr_vec = NULL;
1185 }
1186
1187 static void
1188 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1189 {
1190         int i;
1191
1192         PMD_INIT_FUNC_TRACE();
1193
1194         if (dev->data->tx_queues) {
1195                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1196                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1197
1198                         tx_queue_free(txq);
1199                 }
1200         }
1201
1202         if (dev->data->rx_queues) {
1203                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1204                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1205         }
1206 }
1207
1208 static void
1209 fm10k_dev_close(struct rte_eth_dev *dev)
1210 {
1211         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1212
1213         PMD_INIT_FUNC_TRACE();
1214
1215         fm10k_mbx_lock(hw);
1216         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1217                 MAX_LPORT_NUM, false);
1218         fm10k_mbx_unlock(hw);
1219
1220         /* allow 100ms for device to quiesce */
1221         rte_delay_us(FM10K_SWITCH_QUIESCE_US);
1222
1223         /* Stop mailbox service first */
1224         fm10k_close_mbx_service(hw);
1225         fm10k_dev_stop(dev);
1226         fm10k_dev_queue_release(dev);
1227         fm10k_stop_hw(hw);
1228 }
1229
1230 static int
1231 fm10k_link_update(struct rte_eth_dev *dev,
1232         __rte_unused int wait_to_complete)
1233 {
1234         struct fm10k_dev_info *dev_info =
1235                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1236         PMD_INIT_FUNC_TRACE();
1237
1238         dev->data->dev_link.link_speed  = ETH_SPEED_NUM_50G;
1239         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1240         dev->data->dev_link.link_status =
1241                 dev_info->sm_down ? ETH_LINK_DOWN : ETH_LINK_UP;
1242         dev->data->dev_link.link_autoneg = ETH_LINK_FIXED;
1243
1244         return 0;
1245 }
1246
1247 static int fm10k_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1248         struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit)
1249 {
1250         unsigned i, q;
1251         unsigned count = 0;
1252
1253         if (xstats_names != NULL) {
1254                 /* Note: limit checked in rte_eth_xstats_names() */
1255
1256                 /* Global stats */
1257                 for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1258                         snprintf(xstats_names[count].name,
1259                                 sizeof(xstats_names[count].name),
1260                                 "%s", fm10k_hw_stats_strings[count].name);
1261                         count++;
1262                 }
1263
1264                 /* PF queue stats */
1265                 for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1266                         for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1267                                 snprintf(xstats_names[count].name,
1268                                         sizeof(xstats_names[count].name),
1269                                         "rx_q%u_%s", q,
1270                                         fm10k_hw_stats_rx_q_strings[i].name);
1271                                 count++;
1272                         }
1273                         for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1274                                 snprintf(xstats_names[count].name,
1275                                         sizeof(xstats_names[count].name),
1276                                         "tx_q%u_%s", q,
1277                                         fm10k_hw_stats_tx_q_strings[i].name);
1278                                 count++;
1279                         }
1280                 }
1281         }
1282         return FM10K_NB_XSTATS;
1283 }
1284
1285 static int
1286 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1287                  unsigned n)
1288 {
1289         struct fm10k_hw_stats *hw_stats =
1290                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1291         unsigned i, q, count = 0;
1292
1293         if (n < FM10K_NB_XSTATS)
1294                 return FM10K_NB_XSTATS;
1295
1296         /* Global stats */
1297         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1298                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1299                         fm10k_hw_stats_strings[count].offset);
1300                 xstats[count].id = count;
1301                 count++;
1302         }
1303
1304         /* PF queue stats */
1305         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1306                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1307                         xstats[count].value =
1308                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1309                                 fm10k_hw_stats_rx_q_strings[i].offset);
1310                         xstats[count].id = count;
1311                         count++;
1312                 }
1313                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1314                         xstats[count].value =
1315                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1316                                 fm10k_hw_stats_tx_q_strings[i].offset);
1317                         xstats[count].id = count;
1318                         count++;
1319                 }
1320         }
1321
1322         return FM10K_NB_XSTATS;
1323 }
1324
1325 static int
1326 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1327 {
1328         uint64_t ipackets, opackets, ibytes, obytes;
1329         struct fm10k_hw *hw =
1330                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1331         struct fm10k_hw_stats *hw_stats =
1332                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1333         int i;
1334
1335         PMD_INIT_FUNC_TRACE();
1336
1337         fm10k_update_hw_stats(hw, hw_stats);
1338
1339         ipackets = opackets = ibytes = obytes = 0;
1340         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1341                 (i < hw->mac.max_queues); ++i) {
1342                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1343                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1344                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1345                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1346                 ipackets += stats->q_ipackets[i];
1347                 opackets += stats->q_opackets[i];
1348                 ibytes   += stats->q_ibytes[i];
1349                 obytes   += stats->q_obytes[i];
1350         }
1351         stats->ipackets = ipackets;
1352         stats->opackets = opackets;
1353         stats->ibytes = ibytes;
1354         stats->obytes = obytes;
1355         return 0;
1356 }
1357
1358 static void
1359 fm10k_stats_reset(struct rte_eth_dev *dev)
1360 {
1361         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1362         struct fm10k_hw_stats *hw_stats =
1363                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1364
1365         PMD_INIT_FUNC_TRACE();
1366
1367         memset(hw_stats, 0, sizeof(*hw_stats));
1368         fm10k_rebind_hw_stats(hw, hw_stats);
1369 }
1370
1371 static void
1372 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1373         struct rte_eth_dev_info *dev_info)
1374 {
1375         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1376         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1377
1378         PMD_INIT_FUNC_TRACE();
1379
1380         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1381         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1382         dev_info->max_rx_queues      = hw->mac.max_queues;
1383         dev_info->max_tx_queues      = hw->mac.max_queues;
1384         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1385         dev_info->max_hash_mac_addrs = 0;
1386         dev_info->max_vfs            = pdev->max_vfs;
1387         dev_info->vmdq_pool_base     = 0;
1388         dev_info->vmdq_queue_base    = 0;
1389         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1390         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1391         dev_info->rx_queue_offload_capa = fm10k_get_rx_queue_offloads_capa(dev);
1392         dev_info->rx_offload_capa = fm10k_get_rx_port_offloads_capa(dev) |
1393                                     dev_info->rx_queue_offload_capa;
1394         dev_info->tx_queue_offload_capa = fm10k_get_tx_queue_offloads_capa(dev);
1395         dev_info->tx_offload_capa = fm10k_get_tx_port_offloads_capa(dev) |
1396                                     dev_info->tx_queue_offload_capa;
1397
1398         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1399         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1400
1401         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1402                 .rx_thresh = {
1403                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1404                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1405                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1406                 },
1407                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1408                 .rx_drop_en = 0,
1409                 .offloads = 0,
1410         };
1411
1412         dev_info->default_txconf = (struct rte_eth_txconf) {
1413                 .tx_thresh = {
1414                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1415                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1416                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1417                 },
1418                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1419                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1420                 .offloads = 0,
1421         };
1422
1423         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1424                 .nb_max = FM10K_MAX_RX_DESC,
1425                 .nb_min = FM10K_MIN_RX_DESC,
1426                 .nb_align = FM10K_MULT_RX_DESC,
1427         };
1428
1429         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1430                 .nb_max = FM10K_MAX_TX_DESC,
1431                 .nb_min = FM10K_MIN_TX_DESC,
1432                 .nb_align = FM10K_MULT_TX_DESC,
1433                 .nb_seg_max = FM10K_TX_MAX_SEG,
1434                 .nb_mtu_seg_max = FM10K_TX_MAX_MTU_SEG,
1435         };
1436
1437         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
1438                         ETH_LINK_SPEED_10G | ETH_LINK_SPEED_25G |
1439                         ETH_LINK_SPEED_40G | ETH_LINK_SPEED_100G;
1440 }
1441
1442 #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
1443 static const uint32_t *
1444 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1445 {
1446         if (dev->rx_pkt_burst == fm10k_recv_pkts ||
1447             dev->rx_pkt_burst == fm10k_recv_scattered_pkts) {
1448                 static uint32_t ptypes[] = {
1449                         /* refers to rx_desc_to_ol_flags() */
1450                         RTE_PTYPE_L2_ETHER,
1451                         RTE_PTYPE_L3_IPV4,
1452                         RTE_PTYPE_L3_IPV4_EXT,
1453                         RTE_PTYPE_L3_IPV6,
1454                         RTE_PTYPE_L3_IPV6_EXT,
1455                         RTE_PTYPE_L4_TCP,
1456                         RTE_PTYPE_L4_UDP,
1457                         RTE_PTYPE_UNKNOWN
1458                 };
1459
1460                 return ptypes;
1461         } else if (dev->rx_pkt_burst == fm10k_recv_pkts_vec ||
1462                    dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec) {
1463                 static uint32_t ptypes_vec[] = {
1464                         /* refers to fm10k_desc_to_pktype_v() */
1465                         RTE_PTYPE_L3_IPV4,
1466                         RTE_PTYPE_L3_IPV4_EXT,
1467                         RTE_PTYPE_L3_IPV6,
1468                         RTE_PTYPE_L3_IPV6_EXT,
1469                         RTE_PTYPE_L4_TCP,
1470                         RTE_PTYPE_L4_UDP,
1471                         RTE_PTYPE_TUNNEL_GENEVE,
1472                         RTE_PTYPE_TUNNEL_NVGRE,
1473                         RTE_PTYPE_TUNNEL_VXLAN,
1474                         RTE_PTYPE_TUNNEL_GRE,
1475                         RTE_PTYPE_UNKNOWN
1476                 };
1477
1478                 return ptypes_vec;
1479         }
1480
1481         return NULL;
1482 }
1483 #else
1484 static const uint32_t *
1485 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1486 {
1487         return NULL;
1488 }
1489 #endif
1490
1491 static int
1492 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1493 {
1494         s32 result;
1495         uint16_t mac_num = 0;
1496         uint32_t vid_idx, vid_bit, mac_index;
1497         struct fm10k_hw *hw;
1498         struct fm10k_macvlan_filter_info *macvlan;
1499         struct rte_eth_dev_data *data = dev->data;
1500
1501         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1502         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1503
1504         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1505                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1506                 return -EINVAL;
1507         }
1508
1509         if (vlan_id > ETH_VLAN_ID_MAX) {
1510                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1511                 return -EINVAL;
1512         }
1513
1514         vid_idx = FM10K_VFTA_IDX(vlan_id);
1515         vid_bit = FM10K_VFTA_BIT(vlan_id);
1516         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1517         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1518                 return 0;
1519         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1520         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1521                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1522                         "in the VLAN filter table");
1523                 return -EINVAL;
1524         }
1525
1526         fm10k_mbx_lock(hw);
1527         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1528         fm10k_mbx_unlock(hw);
1529         if (result != FM10K_SUCCESS) {
1530                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1531                 return -EIO;
1532         }
1533
1534         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1535                         (result == FM10K_SUCCESS); mac_index++) {
1536                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1537                         continue;
1538                 if (mac_num > macvlan->mac_num - 1) {
1539                         PMD_INIT_LOG(ERR, "MAC address number "
1540                                         "not match");
1541                         break;
1542                 }
1543                 fm10k_mbx_lock(hw);
1544                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1545                         data->mac_addrs[mac_index].addr_bytes,
1546                         vlan_id, on, 0);
1547                 fm10k_mbx_unlock(hw);
1548                 mac_num++;
1549         }
1550         if (result != FM10K_SUCCESS) {
1551                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1552                 return -EIO;
1553         }
1554
1555         if (on) {
1556                 macvlan->vlan_num++;
1557                 macvlan->vfta[vid_idx] |= vid_bit;
1558         } else {
1559                 macvlan->vlan_num--;
1560                 macvlan->vfta[vid_idx] &= ~vid_bit;
1561         }
1562         return 0;
1563 }
1564
1565 static int
1566 fm10k_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1567 {
1568         if (mask & ETH_VLAN_STRIP_MASK) {
1569                 if (!(dev->data->dev_conf.rxmode.offloads &
1570                         DEV_RX_OFFLOAD_VLAN_STRIP))
1571                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1572                                         "always on in fm10k");
1573         }
1574
1575         if (mask & ETH_VLAN_EXTEND_MASK) {
1576                 if (dev->data->dev_conf.rxmode.offloads &
1577                         DEV_RX_OFFLOAD_VLAN_EXTEND)
1578                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1579                                         "supported in fm10k");
1580         }
1581
1582         if (mask & ETH_VLAN_FILTER_MASK) {
1583                 if (!(dev->data->dev_conf.rxmode.offloads &
1584                         DEV_RX_OFFLOAD_VLAN_FILTER))
1585                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1586         }
1587
1588         return 0;
1589 }
1590
1591 /* Add/Remove a MAC address, and update filters to main VSI */
1592 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1593                 const u8 *mac, bool add, uint32_t pool)
1594 {
1595         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1596         struct fm10k_macvlan_filter_info *macvlan;
1597         uint32_t i, j, k;
1598
1599         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1600
1601         if (pool != MAIN_VSI_POOL_NUMBER) {
1602                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1603                         "mac to pool %u", pool);
1604                 return;
1605         }
1606         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1607                 if (!macvlan->vfta[j])
1608                         continue;
1609                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1610                         if (!(macvlan->vfta[j] & (1 << k)))
1611                                 continue;
1612                         if (i + 1 > macvlan->vlan_num) {
1613                                 PMD_INIT_LOG(ERR, "vlan number not match");
1614                                 return;
1615                         }
1616                         fm10k_mbx_lock(hw);
1617                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1618                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1619                         fm10k_mbx_unlock(hw);
1620                         i++;
1621                 }
1622         }
1623 }
1624
1625 /* Add/Remove a MAC address, and update filters to VMDQ */
1626 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1627                 const u8 *mac, bool add, uint32_t pool)
1628 {
1629         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1630         struct fm10k_macvlan_filter_info *macvlan;
1631         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1632         uint32_t i;
1633
1634         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1635         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1636
1637         if (pool > macvlan->nb_queue_pools) {
1638                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1639                         " Max pool is %u",
1640                         pool, macvlan->nb_queue_pools);
1641                 return;
1642         }
1643         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1644                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1645                         continue;
1646                 fm10k_mbx_lock(hw);
1647                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1648                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1649                 fm10k_mbx_unlock(hw);
1650         }
1651 }
1652
1653 /* Add/Remove a MAC address, and update filters */
1654 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1655                 const u8 *mac, bool add, uint32_t pool)
1656 {
1657         struct fm10k_macvlan_filter_info *macvlan;
1658
1659         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1660
1661         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1662                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1663         else
1664                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1665
1666         if (add)
1667                 macvlan->mac_num++;
1668         else
1669                 macvlan->mac_num--;
1670 }
1671
1672 /* Add a MAC address, and update filters */
1673 static int
1674 fm10k_macaddr_add(struct rte_eth_dev *dev,
1675                 struct ether_addr *mac_addr,
1676                 uint32_t index,
1677                 uint32_t pool)
1678 {
1679         struct fm10k_macvlan_filter_info *macvlan;
1680
1681         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1682         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1683         macvlan->mac_vmdq_id[index] = pool;
1684         return 0;
1685 }
1686
1687 /* Remove a MAC address, and update filters */
1688 static void
1689 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1690 {
1691         struct rte_eth_dev_data *data = dev->data;
1692         struct fm10k_macvlan_filter_info *macvlan;
1693
1694         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1695         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1696                         FALSE, macvlan->mac_vmdq_id[index]);
1697         macvlan->mac_vmdq_id[index] = 0;
1698 }
1699
1700 static inline int
1701 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1702 {
1703         if ((request < min) || (request > max) || ((request % mult) != 0))
1704                 return -1;
1705         else
1706                 return 0;
1707 }
1708
1709
1710 static inline int
1711 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1712 {
1713         if ((request < min) || (request > max) || ((div % request) != 0))
1714                 return -1;
1715         else
1716                 return 0;
1717 }
1718
1719 static inline int
1720 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1721 {
1722         uint16_t rx_free_thresh;
1723
1724         if (conf->rx_free_thresh == 0)
1725                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1726         else
1727                 rx_free_thresh = conf->rx_free_thresh;
1728
1729         /* make sure the requested threshold satisfies the constraints */
1730         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1731                         FM10K_RX_FREE_THRESH_MAX(q),
1732                         FM10K_RX_FREE_THRESH_DIV(q),
1733                         rx_free_thresh)) {
1734                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1735                         "less than or equal to %u, "
1736                         "greater than or equal to %u, "
1737                         "and a divisor of %u",
1738                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1739                         FM10K_RX_FREE_THRESH_MIN(q),
1740                         FM10K_RX_FREE_THRESH_DIV(q));
1741                 return -EINVAL;
1742         }
1743
1744         q->alloc_thresh = rx_free_thresh;
1745         q->drop_en = conf->rx_drop_en;
1746         q->rx_deferred_start = conf->rx_deferred_start;
1747
1748         return 0;
1749 }
1750
1751 /*
1752  * Hardware requires specific alignment for Rx packet buffers. At
1753  * least one of the following two conditions must be satisfied.
1754  *  1. Address is 512B aligned
1755  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1756  *
1757  * As such, the driver may need to adjust the DMA address within the
1758  * buffer by up to 512B.
1759  *
1760  * return 1 if the element size is valid, otherwise return 0.
1761  */
1762 static int
1763 mempool_element_size_valid(struct rte_mempool *mp)
1764 {
1765         uint32_t min_size;
1766
1767         /* elt_size includes mbuf header and headroom */
1768         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1769                         RTE_PKTMBUF_HEADROOM;
1770
1771         /* account for up to 512B of alignment */
1772         min_size -= FM10K_RX_DATABUF_ALIGN;
1773
1774         /* sanity check for overflow */
1775         if (min_size > mp->elt_size)
1776                 return 0;
1777
1778         /* size is valid */
1779         return 1;
1780 }
1781
1782 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev)
1783 {
1784         RTE_SET_USED(dev);
1785
1786         return (uint64_t)(DEV_RX_OFFLOAD_SCATTER);
1787 }
1788
1789 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev)
1790 {
1791         RTE_SET_USED(dev);
1792
1793         return  (uint64_t)(DEV_RX_OFFLOAD_VLAN_STRIP  |
1794                            DEV_RX_OFFLOAD_VLAN_FILTER |
1795                            DEV_RX_OFFLOAD_IPV4_CKSUM  |
1796                            DEV_RX_OFFLOAD_UDP_CKSUM   |
1797                            DEV_RX_OFFLOAD_TCP_CKSUM   |
1798                            DEV_RX_OFFLOAD_JUMBO_FRAME |
1799                            DEV_RX_OFFLOAD_CRC_STRIP   |
1800                            DEV_RX_OFFLOAD_HEADER_SPLIT);
1801 }
1802
1803 static int
1804 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1805         uint16_t nb_desc, unsigned int socket_id,
1806         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1807 {
1808         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1809         struct fm10k_dev_info *dev_info =
1810                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1811         struct fm10k_rx_queue *q;
1812         const struct rte_memzone *mz;
1813         uint64_t offloads;
1814
1815         PMD_INIT_FUNC_TRACE();
1816
1817         offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads;
1818
1819         /* make sure the mempool element size can account for alignment. */
1820         if (!mempool_element_size_valid(mp)) {
1821                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1822                 return -EINVAL;
1823         }
1824
1825         /* make sure a valid number of descriptors have been requested */
1826         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1827                                 FM10K_MULT_RX_DESC, nb_desc)) {
1828                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1829                         "less than or equal to %"PRIu32", "
1830                         "greater than or equal to %u, "
1831                         "and a multiple of %u",
1832                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1833                         FM10K_MULT_RX_DESC);
1834                 return -EINVAL;
1835         }
1836
1837         /*
1838          * if this queue existed already, free the associated memory. The
1839          * queue cannot be reused in case we need to allocate memory on
1840          * different socket than was previously used.
1841          */
1842         if (dev->data->rx_queues[queue_id] != NULL) {
1843                 rx_queue_free(dev->data->rx_queues[queue_id]);
1844                 dev->data->rx_queues[queue_id] = NULL;
1845         }
1846
1847         /* allocate memory for the queue structure */
1848         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1849                                 socket_id);
1850         if (q == NULL) {
1851                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1852                 return -ENOMEM;
1853         }
1854
1855         /* setup queue */
1856         q->mp = mp;
1857         q->nb_desc = nb_desc;
1858         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1859         q->port_id = dev->data->port_id;
1860         q->queue_id = queue_id;
1861         q->tail_ptr = (volatile uint32_t *)
1862                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1863         q->offloads = offloads;
1864         if (handle_rxconf(q, conf))
1865                 return -EINVAL;
1866
1867         /* allocate memory for the software ring */
1868         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1869                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1870                         RTE_CACHE_LINE_SIZE, socket_id);
1871         if (q->sw_ring == NULL) {
1872                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1873                 rte_free(q);
1874                 return -ENOMEM;
1875         }
1876
1877         /*
1878          * allocate memory for the hardware descriptor ring. A memzone large
1879          * enough to hold the maximum ring size is requested to allow for
1880          * resizing in later calls to the queue setup function.
1881          */
1882         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1883                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1884                                       socket_id);
1885         if (mz == NULL) {
1886                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1887                 rte_free(q->sw_ring);
1888                 rte_free(q);
1889                 return -ENOMEM;
1890         }
1891         q->hw_ring = mz->addr;
1892         q->hw_ring_phys_addr = mz->iova;
1893
1894         /* Check if number of descs satisfied Vector requirement */
1895         if (!rte_is_power_of_2(nb_desc)) {
1896                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1897                                     "preconditions - canceling the feature for "
1898                                     "the whole port[%d]",
1899                              q->queue_id, q->port_id);
1900                 dev_info->rx_vec_allowed = false;
1901         } else
1902                 fm10k_rxq_vec_setup(q);
1903
1904         dev->data->rx_queues[queue_id] = q;
1905         return 0;
1906 }
1907
1908 static void
1909 fm10k_rx_queue_release(void *queue)
1910 {
1911         PMD_INIT_FUNC_TRACE();
1912
1913         rx_queue_free(queue);
1914 }
1915
1916 static inline int
1917 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1918 {
1919         uint16_t tx_free_thresh;
1920         uint16_t tx_rs_thresh;
1921
1922         /* constraint MACROs require that tx_free_thresh is configured
1923          * before tx_rs_thresh */
1924         if (conf->tx_free_thresh == 0)
1925                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1926         else
1927                 tx_free_thresh = conf->tx_free_thresh;
1928
1929         /* make sure the requested threshold satisfies the constraints */
1930         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1931                         FM10K_TX_FREE_THRESH_MAX(q),
1932                         FM10K_TX_FREE_THRESH_DIV(q),
1933                         tx_free_thresh)) {
1934                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1935                         "less than or equal to %u, "
1936                         "greater than or equal to %u, "
1937                         "and a divisor of %u",
1938                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1939                         FM10K_TX_FREE_THRESH_MIN(q),
1940                         FM10K_TX_FREE_THRESH_DIV(q));
1941                 return -EINVAL;
1942         }
1943
1944         q->free_thresh = tx_free_thresh;
1945
1946         if (conf->tx_rs_thresh == 0)
1947                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1948         else
1949                 tx_rs_thresh = conf->tx_rs_thresh;
1950
1951         q->tx_deferred_start = conf->tx_deferred_start;
1952
1953         /* make sure the requested threshold satisfies the constraints */
1954         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1955                         FM10K_TX_RS_THRESH_MAX(q),
1956                         FM10K_TX_RS_THRESH_DIV(q),
1957                         tx_rs_thresh)) {
1958                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1959                         "less than or equal to %u, "
1960                         "greater than or equal to %u, "
1961                         "and a divisor of %u",
1962                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1963                         FM10K_TX_RS_THRESH_MIN(q),
1964                         FM10K_TX_RS_THRESH_DIV(q));
1965                 return -EINVAL;
1966         }
1967
1968         q->rs_thresh = tx_rs_thresh;
1969
1970         return 0;
1971 }
1972
1973 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev)
1974 {
1975         RTE_SET_USED(dev);
1976
1977         return 0;
1978 }
1979
1980 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev)
1981 {
1982         RTE_SET_USED(dev);
1983
1984         return (uint64_t)(DEV_TX_OFFLOAD_VLAN_INSERT |
1985                           DEV_TX_OFFLOAD_IPV4_CKSUM  |
1986                           DEV_TX_OFFLOAD_UDP_CKSUM   |
1987                           DEV_TX_OFFLOAD_TCP_CKSUM   |
1988                           DEV_TX_OFFLOAD_TCP_TSO);
1989 }
1990
1991 static int
1992 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1993         uint16_t nb_desc, unsigned int socket_id,
1994         const struct rte_eth_txconf *conf)
1995 {
1996         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1997         struct fm10k_tx_queue *q;
1998         const struct rte_memzone *mz;
1999         uint64_t offloads;
2000
2001         PMD_INIT_FUNC_TRACE();
2002
2003         offloads = conf->offloads | dev->data->dev_conf.txmode.offloads;
2004
2005         /* make sure a valid number of descriptors have been requested */
2006         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
2007                                 FM10K_MULT_TX_DESC, nb_desc)) {
2008                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
2009                         "less than or equal to %"PRIu32", "
2010                         "greater than or equal to %u, "
2011                         "and a multiple of %u",
2012                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
2013                         FM10K_MULT_TX_DESC);
2014                 return -EINVAL;
2015         }
2016
2017         /*
2018          * if this queue existed already, free the associated memory. The
2019          * queue cannot be reused in case we need to allocate memory on
2020          * different socket than was previously used.
2021          */
2022         if (dev->data->tx_queues[queue_id] != NULL) {
2023                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
2024
2025                 tx_queue_free(txq);
2026                 dev->data->tx_queues[queue_id] = NULL;
2027         }
2028
2029         /* allocate memory for the queue structure */
2030         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
2031                                 socket_id);
2032         if (q == NULL) {
2033                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
2034                 return -ENOMEM;
2035         }
2036
2037         /* setup queue */
2038         q->nb_desc = nb_desc;
2039         q->port_id = dev->data->port_id;
2040         q->queue_id = queue_id;
2041         q->offloads = offloads;
2042         q->ops = &def_txq_ops;
2043         q->tail_ptr = (volatile uint32_t *)
2044                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
2045         if (handle_txconf(q, conf))
2046                 return -EINVAL;
2047
2048         /* allocate memory for the software ring */
2049         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
2050                                         nb_desc * sizeof(struct rte_mbuf *),
2051                                         RTE_CACHE_LINE_SIZE, socket_id);
2052         if (q->sw_ring == NULL) {
2053                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
2054                 rte_free(q);
2055                 return -ENOMEM;
2056         }
2057
2058         /*
2059          * allocate memory for the hardware descriptor ring. A memzone large
2060          * enough to hold the maximum ring size is requested to allow for
2061          * resizing in later calls to the queue setup function.
2062          */
2063         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
2064                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
2065                                       socket_id);
2066         if (mz == NULL) {
2067                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
2068                 rte_free(q->sw_ring);
2069                 rte_free(q);
2070                 return -ENOMEM;
2071         }
2072         q->hw_ring = mz->addr;
2073         q->hw_ring_phys_addr = mz->iova;
2074
2075         /*
2076          * allocate memory for the RS bit tracker. Enough slots to hold the
2077          * descriptor index for each RS bit needing to be set are required.
2078          */
2079         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
2080                                 ((nb_desc + 1) / q->rs_thresh) *
2081                                 sizeof(uint16_t),
2082                                 RTE_CACHE_LINE_SIZE, socket_id);
2083         if (q->rs_tracker.list == NULL) {
2084                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
2085                 rte_free(q->sw_ring);
2086                 rte_free(q);
2087                 return -ENOMEM;
2088         }
2089
2090         dev->data->tx_queues[queue_id] = q;
2091         return 0;
2092 }
2093
2094 static void
2095 fm10k_tx_queue_release(void *queue)
2096 {
2097         struct fm10k_tx_queue *q = queue;
2098         PMD_INIT_FUNC_TRACE();
2099
2100         tx_queue_free(q);
2101 }
2102
2103 static int
2104 fm10k_reta_update(struct rte_eth_dev *dev,
2105                         struct rte_eth_rss_reta_entry64 *reta_conf,
2106                         uint16_t reta_size)
2107 {
2108         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2109         uint16_t i, j, idx, shift;
2110         uint8_t mask;
2111         uint32_t reta;
2112
2113         PMD_INIT_FUNC_TRACE();
2114
2115         if (reta_size > FM10K_MAX_RSS_INDICES) {
2116                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2117                         "(%d) doesn't match the number hardware can supported "
2118                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2119                 return -EINVAL;
2120         }
2121
2122         /*
2123          * Update Redirection Table RETA[n], n=0..31. The redirection table has
2124          * 128-entries in 32 registers
2125          */
2126         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2127                 idx = i / RTE_RETA_GROUP_SIZE;
2128                 shift = i % RTE_RETA_GROUP_SIZE;
2129                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2130                                 BIT_MASK_PER_UINT32);
2131                 if (mask == 0)
2132                         continue;
2133
2134                 reta = 0;
2135                 if (mask != BIT_MASK_PER_UINT32)
2136                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2137
2138                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2139                         if (mask & (0x1 << j)) {
2140                                 if (mask != 0xF)
2141                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
2142                                 reta |= reta_conf[idx].reta[shift + j] <<
2143                                                 (CHAR_BIT * j);
2144                         }
2145                 }
2146                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2147         }
2148
2149         return 0;
2150 }
2151
2152 static int
2153 fm10k_reta_query(struct rte_eth_dev *dev,
2154                         struct rte_eth_rss_reta_entry64 *reta_conf,
2155                         uint16_t reta_size)
2156 {
2157         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2158         uint16_t i, j, idx, shift;
2159         uint8_t mask;
2160         uint32_t reta;
2161
2162         PMD_INIT_FUNC_TRACE();
2163
2164         if (reta_size < FM10K_MAX_RSS_INDICES) {
2165                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2166                         "(%d) doesn't match the number hardware can supported "
2167                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2168                 return -EINVAL;
2169         }
2170
2171         /*
2172          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2173          * 128-entries in 32 registers
2174          */
2175         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2176                 idx = i / RTE_RETA_GROUP_SIZE;
2177                 shift = i % RTE_RETA_GROUP_SIZE;
2178                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2179                                 BIT_MASK_PER_UINT32);
2180                 if (mask == 0)
2181                         continue;
2182
2183                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2184                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2185                         if (mask & (0x1 << j))
2186                                 reta_conf[idx].reta[shift + j] = ((reta >>
2187                                         CHAR_BIT * j) & UINT8_MAX);
2188                 }
2189         }
2190
2191         return 0;
2192 }
2193
2194 static int
2195 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2196         struct rte_eth_rss_conf *rss_conf)
2197 {
2198         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2199         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2200         uint32_t mrqc;
2201         uint64_t hf = rss_conf->rss_hf;
2202         int i;
2203
2204         PMD_INIT_FUNC_TRACE();
2205
2206         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2207                                 FM10K_RSSRK_ENTRIES_PER_REG))
2208                 return -EINVAL;
2209
2210         if (hf == 0)
2211                 return -EINVAL;
2212
2213         mrqc = 0;
2214         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2215         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2216         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2217         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2218         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2219         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2220         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2221         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2222         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2223
2224         /* If the mapping doesn't fit any supported, return */
2225         if (mrqc == 0)
2226                 return -EINVAL;
2227
2228         if (key != NULL)
2229                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2230                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2231
2232         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2233
2234         return 0;
2235 }
2236
2237 static int
2238 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2239         struct rte_eth_rss_conf *rss_conf)
2240 {
2241         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2242         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2243         uint32_t mrqc;
2244         uint64_t hf;
2245         int i;
2246
2247         PMD_INIT_FUNC_TRACE();
2248
2249         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2250                                 FM10K_RSSRK_ENTRIES_PER_REG))
2251                 return -EINVAL;
2252
2253         if (key != NULL)
2254                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2255                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2256
2257         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2258         hf = 0;
2259         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2260         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2261         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2262         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2263         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2264         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2265         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2266         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2267         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2268
2269         rss_conf->rss_hf = hf;
2270
2271         return 0;
2272 }
2273
2274 static void
2275 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2276 {
2277         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2278         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2279
2280         /* Bind all local non-queue interrupt to vector 0 */
2281         int_map |= FM10K_MISC_VEC_ID;
2282
2283         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2284         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2285         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2286         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2287         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2288         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2289
2290         /* Enable misc causes */
2291         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2292                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2293                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2294                                 FM10K_EIMR_ENABLE(MAILBOX) |
2295                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2296                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2297                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2298                                 FM10K_EIMR_ENABLE(VFLR));
2299
2300         /* Enable ITR 0 */
2301         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2302                                         FM10K_ITR_MASK_CLEAR);
2303         FM10K_WRITE_FLUSH(hw);
2304 }
2305
2306 static void
2307 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2308 {
2309         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2310         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2311
2312         int_map |= FM10K_MISC_VEC_ID;
2313
2314         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2315         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2316         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2317         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2318         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2319         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2320
2321         /* Disable misc causes */
2322         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2323                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2324                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2325                                 FM10K_EIMR_DISABLE(MAILBOX) |
2326                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2327                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2328                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2329                                 FM10K_EIMR_DISABLE(VFLR));
2330
2331         /* Disable ITR 0 */
2332         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2333         FM10K_WRITE_FLUSH(hw);
2334 }
2335
2336 static void
2337 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2338 {
2339         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2340         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2341
2342         /* Bind all local non-queue interrupt to vector 0 */
2343         int_map |= FM10K_MISC_VEC_ID;
2344
2345         /* Only INT 0 available, other 15 are reserved. */
2346         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2347
2348         /* Enable ITR 0 */
2349         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2350                                         FM10K_ITR_MASK_CLEAR);
2351         FM10K_WRITE_FLUSH(hw);
2352 }
2353
2354 static void
2355 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2356 {
2357         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2358         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2359
2360         int_map |= FM10K_MISC_VEC_ID;
2361
2362         /* Only INT 0 available, other 15 are reserved. */
2363         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2364
2365         /* Disable ITR 0 */
2366         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2367         FM10K_WRITE_FLUSH(hw);
2368 }
2369
2370 static int
2371 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2372 {
2373         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2374         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2375
2376         /* Enable ITR */
2377         if (hw->mac.type == fm10k_mac_pf)
2378                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2379                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2380         else
2381                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2382                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2383         rte_intr_enable(&pdev->intr_handle);
2384         return 0;
2385 }
2386
2387 static int
2388 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2389 {
2390         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2391         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2392
2393         /* Disable ITR */
2394         if (hw->mac.type == fm10k_mac_pf)
2395                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2396                         FM10K_ITR_MASK_SET);
2397         else
2398                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2399                         FM10K_ITR_MASK_SET);
2400         return 0;
2401 }
2402
2403 static int
2404 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2405 {
2406         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2407         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2408         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2409         uint32_t intr_vector, vec;
2410         uint16_t queue_id;
2411         int result = 0;
2412
2413         /* fm10k needs one separate interrupt for mailbox,
2414          * so only drivers which support multiple interrupt vectors
2415          * e.g. vfio-pci can work for fm10k interrupt mode
2416          */
2417         if (!rte_intr_cap_multiple(intr_handle) ||
2418                         dev->data->dev_conf.intr_conf.rxq == 0)
2419                 return result;
2420
2421         intr_vector = dev->data->nb_rx_queues;
2422
2423         /* disable interrupt first */
2424         rte_intr_disable(intr_handle);
2425         if (hw->mac.type == fm10k_mac_pf)
2426                 fm10k_dev_disable_intr_pf(dev);
2427         else
2428                 fm10k_dev_disable_intr_vf(dev);
2429
2430         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2431                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2432                 result = -EIO;
2433         }
2434
2435         if (rte_intr_dp_is_en(intr_handle) && !result) {
2436                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2437                         dev->data->nb_rx_queues * sizeof(int), 0);
2438                 if (intr_handle->intr_vec) {
2439                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2440                                         queue_id < dev->data->nb_rx_queues;
2441                                         queue_id++) {
2442                                 intr_handle->intr_vec[queue_id] = vec;
2443                                 if (vec < intr_handle->nb_efd - 1
2444                                                 + FM10K_RX_VEC_START)
2445                                         vec++;
2446                         }
2447                 } else {
2448                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2449                                 " intr_vec", dev->data->nb_rx_queues);
2450                         rte_intr_efd_disable(intr_handle);
2451                         result = -ENOMEM;
2452                 }
2453         }
2454
2455         if (hw->mac.type == fm10k_mac_pf)
2456                 fm10k_dev_enable_intr_pf(dev);
2457         else
2458                 fm10k_dev_enable_intr_vf(dev);
2459         rte_intr_enable(intr_handle);
2460         hw->mac.ops.update_int_moderator(hw);
2461         return result;
2462 }
2463
2464 static int
2465 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2466 {
2467         struct fm10k_fault fault;
2468         int err;
2469         const char *estr = "Unknown error";
2470
2471         /* Process PCA fault */
2472         if (eicr & FM10K_EICR_PCA_FAULT) {
2473                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2474                 if (err)
2475                         goto error;
2476                 switch (fault.type) {
2477                 case PCA_NO_FAULT:
2478                         estr = "PCA_NO_FAULT"; break;
2479                 case PCA_UNMAPPED_ADDR:
2480                         estr = "PCA_UNMAPPED_ADDR"; break;
2481                 case PCA_BAD_QACCESS_PF:
2482                         estr = "PCA_BAD_QACCESS_PF"; break;
2483                 case PCA_BAD_QACCESS_VF:
2484                         estr = "PCA_BAD_QACCESS_VF"; break;
2485                 case PCA_MALICIOUS_REQ:
2486                         estr = "PCA_MALICIOUS_REQ"; break;
2487                 case PCA_POISONED_TLP:
2488                         estr = "PCA_POISONED_TLP"; break;
2489                 case PCA_TLP_ABORT:
2490                         estr = "PCA_TLP_ABORT"; break;
2491                 default:
2492                         goto error;
2493                 }
2494                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2495                         estr, fault.func ? "VF" : "PF", fault.func,
2496                         fault.address, fault.specinfo);
2497         }
2498
2499         /* Process THI fault */
2500         if (eicr & FM10K_EICR_THI_FAULT) {
2501                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2502                 if (err)
2503                         goto error;
2504                 switch (fault.type) {
2505                 case THI_NO_FAULT:
2506                         estr = "THI_NO_FAULT"; break;
2507                 case THI_MAL_DIS_Q_FAULT:
2508                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2509                 default:
2510                         goto error;
2511                 }
2512                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2513                         estr, fault.func ? "VF" : "PF", fault.func,
2514                         fault.address, fault.specinfo);
2515         }
2516
2517         /* Process FUM fault */
2518         if (eicr & FM10K_EICR_FUM_FAULT) {
2519                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2520                 if (err)
2521                         goto error;
2522                 switch (fault.type) {
2523                 case FUM_NO_FAULT:
2524                         estr = "FUM_NO_FAULT"; break;
2525                 case FUM_UNMAPPED_ADDR:
2526                         estr = "FUM_UNMAPPED_ADDR"; break;
2527                 case FUM_POISONED_TLP:
2528                         estr = "FUM_POISONED_TLP"; break;
2529                 case FUM_BAD_VF_QACCESS:
2530                         estr = "FUM_BAD_VF_QACCESS"; break;
2531                 case FUM_ADD_DECODE_ERR:
2532                         estr = "FUM_ADD_DECODE_ERR"; break;
2533                 case FUM_RO_ERROR:
2534                         estr = "FUM_RO_ERROR"; break;
2535                 case FUM_QPRC_CRC_ERROR:
2536                         estr = "FUM_QPRC_CRC_ERROR"; break;
2537                 case FUM_CSR_TIMEOUT:
2538                         estr = "FUM_CSR_TIMEOUT"; break;
2539                 case FUM_INVALID_TYPE:
2540                         estr = "FUM_INVALID_TYPE"; break;
2541                 case FUM_INVALID_LENGTH:
2542                         estr = "FUM_INVALID_LENGTH"; break;
2543                 case FUM_INVALID_BE:
2544                         estr = "FUM_INVALID_BE"; break;
2545                 case FUM_INVALID_ALIGN:
2546                         estr = "FUM_INVALID_ALIGN"; break;
2547                 default:
2548                         goto error;
2549                 }
2550                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2551                         estr, fault.func ? "VF" : "PF", fault.func,
2552                         fault.address, fault.specinfo);
2553         }
2554
2555         return 0;
2556 error:
2557         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2558         return err;
2559 }
2560
2561 /**
2562  * PF interrupt handler triggered by NIC for handling specific interrupt.
2563  *
2564  * @param handle
2565  *  Pointer to interrupt handle.
2566  * @param param
2567  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2568  *
2569  * @return
2570  *  void
2571  */
2572 static void
2573 fm10k_dev_interrupt_handler_pf(void *param)
2574 {
2575         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2576         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2577         uint32_t cause, status;
2578         struct fm10k_dev_info *dev_info =
2579                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2580         int status_mbx;
2581         s32 err;
2582
2583         if (hw->mac.type != fm10k_mac_pf)
2584                 return;
2585
2586         cause = FM10K_READ_REG(hw, FM10K_EICR);
2587
2588         /* Handle PCI fault cases */
2589         if (cause & FM10K_EICR_FAULT_MASK) {
2590                 PMD_INIT_LOG(ERR, "INT: find fault!");
2591                 fm10k_dev_handle_fault(hw, cause);
2592         }
2593
2594         /* Handle switch up/down */
2595         if (cause & FM10K_EICR_SWITCHNOTREADY)
2596                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2597
2598         if (cause & FM10K_EICR_SWITCHREADY) {
2599                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2600                 if (dev_info->sm_down == 1) {
2601                         fm10k_mbx_lock(hw);
2602
2603                         /* For recreating logical ports */
2604                         status_mbx = hw->mac.ops.update_lport_state(hw,
2605                                         hw->mac.dglort_map, MAX_LPORT_NUM, 1);
2606                         if (status_mbx == FM10K_SUCCESS)
2607                                 PMD_INIT_LOG(INFO,
2608                                         "INT: Recreated Logical port");
2609                         else
2610                                 PMD_INIT_LOG(INFO,
2611                                         "INT: Logical ports weren't recreated");
2612
2613                         status_mbx = hw->mac.ops.update_xcast_mode(hw,
2614                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2615                         if (status_mbx != FM10K_SUCCESS)
2616                                 PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2617
2618                         fm10k_mbx_unlock(hw);
2619
2620                         /* first clear the internal SW recording structure */
2621                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2622                                                 ETH_MQ_RX_VMDQ_FLAG))
2623                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2624                                         false);
2625
2626                         fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2627                                         MAIN_VSI_POOL_NUMBER);
2628
2629                         /*
2630                          * Add default mac address and vlan for the logical
2631                          * ports that have been created, leave to the
2632                          * application to fully recover Rx filtering.
2633                          */
2634                         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2635                                         MAIN_VSI_POOL_NUMBER);
2636
2637                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2638                                                 ETH_MQ_RX_VMDQ_FLAG))
2639                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2640                                         true);
2641
2642                         dev_info->sm_down = 0;
2643                         _rte_eth_dev_callback_process(dev,
2644                                         RTE_ETH_EVENT_INTR_LSC,
2645                                         NULL);
2646                 }
2647         }
2648
2649         /* Handle mailbox message */
2650         fm10k_mbx_lock(hw);
2651         err = hw->mbx.ops.process(hw, &hw->mbx);
2652         fm10k_mbx_unlock(hw);
2653
2654         if (err == FM10K_ERR_RESET_REQUESTED) {
2655                 PMD_INIT_LOG(INFO, "INT: Switch is down");
2656                 dev_info->sm_down = 1;
2657                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2658                                 NULL);
2659         }
2660
2661         /* Handle SRAM error */
2662         if (cause & FM10K_EICR_SRAMERROR) {
2663                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2664
2665                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2666                 /* Write to clear pending bits */
2667                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2668
2669                 /* Todo: print out error message after shared code  updates */
2670         }
2671
2672         /* Clear these 3 events if having any */
2673         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2674                  FM10K_EICR_SWITCHREADY;
2675         if (cause)
2676                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2677
2678         /* Re-enable interrupt from device side */
2679         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2680                                         FM10K_ITR_MASK_CLEAR);
2681         /* Re-enable interrupt from host side */
2682         rte_intr_enable(dev->intr_handle);
2683 }
2684
2685 /**
2686  * VF interrupt handler triggered by NIC for handling specific interrupt.
2687  *
2688  * @param handle
2689  *  Pointer to interrupt handle.
2690  * @param param
2691  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2692  *
2693  * @return
2694  *  void
2695  */
2696 static void
2697 fm10k_dev_interrupt_handler_vf(void *param)
2698 {
2699         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2700         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2701         struct fm10k_mbx_info *mbx = &hw->mbx;
2702         struct fm10k_dev_info *dev_info =
2703                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2704         const enum fm10k_mbx_state state = mbx->state;
2705         int status_mbx;
2706
2707         if (hw->mac.type != fm10k_mac_vf)
2708                 return;
2709
2710         /* Handle mailbox message if lock is acquired */
2711         fm10k_mbx_lock(hw);
2712         hw->mbx.ops.process(hw, &hw->mbx);
2713         fm10k_mbx_unlock(hw);
2714
2715         if (state == FM10K_STATE_OPEN && mbx->state == FM10K_STATE_CONNECT) {
2716                 PMD_INIT_LOG(INFO, "INT: Switch has gone down");
2717
2718                 fm10k_mbx_lock(hw);
2719                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2720                                 MAX_LPORT_NUM, 1);
2721                 fm10k_mbx_unlock(hw);
2722
2723                 /* Setting reset flag */
2724                 dev_info->sm_down = 1;
2725                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2726                                 NULL);
2727         }
2728
2729         if (dev_info->sm_down == 1 &&
2730                         hw->mac.dglort_map == FM10K_DGLORTMAP_ZERO) {
2731                 PMD_INIT_LOG(INFO, "INT: Switch has gone up");
2732                 fm10k_mbx_lock(hw);
2733                 status_mbx = hw->mac.ops.update_xcast_mode(hw,
2734                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2735                 if (status_mbx != FM10K_SUCCESS)
2736                         PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2737                 fm10k_mbx_unlock(hw);
2738
2739                 /* first clear the internal SW recording structure */
2740                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, false);
2741                 fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2742                                 MAIN_VSI_POOL_NUMBER);
2743
2744                 /*
2745                  * Add default mac address and vlan for the logical ports that
2746                  * have been created, leave to the application to fully recover
2747                  * Rx filtering.
2748                  */
2749                 fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2750                                 MAIN_VSI_POOL_NUMBER);
2751                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
2752
2753                 dev_info->sm_down = 0;
2754                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2755                                 NULL);
2756         }
2757
2758         /* Re-enable interrupt from device side */
2759         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2760                                         FM10K_ITR_MASK_CLEAR);
2761         /* Re-enable interrupt from host side */
2762         rte_intr_enable(dev->intr_handle);
2763 }
2764
2765 /* Mailbox message handler in VF */
2766 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2767         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2768         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2769         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2770         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2771 };
2772
2773 static int
2774 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2775 {
2776         int err = 0;
2777
2778         /* Initialize mailbox lock */
2779         fm10k_mbx_initlock(hw);
2780
2781         /* Replace default message handler with new ones */
2782         if (hw->mac.type == fm10k_mac_vf)
2783                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2784
2785         if (err) {
2786                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2787                                 err);
2788                 return err;
2789         }
2790         /* Connect to SM for PF device or PF for VF device */
2791         return hw->mbx.ops.connect(hw, &hw->mbx);
2792 }
2793
2794 static void
2795 fm10k_close_mbx_service(struct fm10k_hw *hw)
2796 {
2797         /* Disconnect from SM for PF device or PF for VF device */
2798         hw->mbx.ops.disconnect(hw, &hw->mbx);
2799 }
2800
2801 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2802         .dev_configure          = fm10k_dev_configure,
2803         .dev_start              = fm10k_dev_start,
2804         .dev_stop               = fm10k_dev_stop,
2805         .dev_close              = fm10k_dev_close,
2806         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2807         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2808         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2809         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2810         .stats_get              = fm10k_stats_get,
2811         .xstats_get             = fm10k_xstats_get,
2812         .xstats_get_names       = fm10k_xstats_get_names,
2813         .stats_reset            = fm10k_stats_reset,
2814         .xstats_reset           = fm10k_stats_reset,
2815         .link_update            = fm10k_link_update,
2816         .dev_infos_get          = fm10k_dev_infos_get,
2817         .dev_supported_ptypes_get = fm10k_dev_supported_ptypes_get,
2818         .vlan_filter_set        = fm10k_vlan_filter_set,
2819         .vlan_offload_set       = fm10k_vlan_offload_set,
2820         .mac_addr_add           = fm10k_macaddr_add,
2821         .mac_addr_remove        = fm10k_macaddr_remove,
2822         .rx_queue_start         = fm10k_dev_rx_queue_start,
2823         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2824         .tx_queue_start         = fm10k_dev_tx_queue_start,
2825         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2826         .rx_queue_setup         = fm10k_rx_queue_setup,
2827         .rx_queue_release       = fm10k_rx_queue_release,
2828         .tx_queue_setup         = fm10k_tx_queue_setup,
2829         .tx_queue_release       = fm10k_tx_queue_release,
2830         .rx_descriptor_done     = fm10k_dev_rx_descriptor_done,
2831         .rx_descriptor_status = fm10k_dev_rx_descriptor_status,
2832         .tx_descriptor_status = fm10k_dev_tx_descriptor_status,
2833         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2834         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2835         .reta_update            = fm10k_reta_update,
2836         .reta_query             = fm10k_reta_query,
2837         .rss_hash_update        = fm10k_rss_hash_update,
2838         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2839 };
2840
2841 static int ftag_check_handler(__rte_unused const char *key,
2842                 const char *value, __rte_unused void *opaque)
2843 {
2844         if (strcmp(value, "1"))
2845                 return -1;
2846
2847         return 0;
2848 }
2849
2850 static int
2851 fm10k_check_ftag(struct rte_devargs *devargs)
2852 {
2853         struct rte_kvargs *kvlist;
2854         const char *ftag_key = "enable_ftag";
2855
2856         if (devargs == NULL)
2857                 return 0;
2858
2859         kvlist = rte_kvargs_parse(devargs->args, NULL);
2860         if (kvlist == NULL)
2861                 return 0;
2862
2863         if (!rte_kvargs_count(kvlist, ftag_key)) {
2864                 rte_kvargs_free(kvlist);
2865                 return 0;
2866         }
2867         /* FTAG is enabled when there's key-value pair: enable_ftag=1 */
2868         if (rte_kvargs_process(kvlist, ftag_key,
2869                                 ftag_check_handler, NULL) < 0) {
2870                 rte_kvargs_free(kvlist);
2871                 return 0;
2872         }
2873         rte_kvargs_free(kvlist);
2874
2875         return 1;
2876 }
2877
2878 static uint16_t
2879 fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
2880                     uint16_t nb_pkts)
2881 {
2882         uint16_t nb_tx = 0;
2883         struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue;
2884
2885         while (nb_pkts) {
2886                 uint16_t ret, num;
2887
2888                 num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh);
2889                 ret = fm10k_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
2890                                                  num);
2891                 nb_tx += ret;
2892                 nb_pkts -= ret;
2893                 if (ret < num)
2894                         break;
2895         }
2896
2897         return nb_tx;
2898 }
2899
2900 static void __attribute__((cold))
2901 fm10k_set_tx_function(struct rte_eth_dev *dev)
2902 {
2903         struct fm10k_tx_queue *txq;
2904         int i;
2905         int use_sse = 1;
2906         uint16_t tx_ftag_en = 0;
2907
2908         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2909                 /* primary process has set the ftag flag and offloads */
2910                 txq = dev->data->tx_queues[0];
2911                 if (fm10k_tx_vec_condition_check(txq)) {
2912                         dev->tx_pkt_burst = fm10k_xmit_pkts;
2913                         dev->tx_pkt_prepare = fm10k_prep_pkts;
2914                         PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2915                 } else {
2916                         PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2917                         dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2918                         dev->tx_pkt_prepare = NULL;
2919                 }
2920                 return;
2921         }
2922
2923         if (fm10k_check_ftag(dev->device->devargs))
2924                 tx_ftag_en = 1;
2925
2926         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2927                 txq = dev->data->tx_queues[i];
2928                 txq->tx_ftag_en = tx_ftag_en;
2929                 /* Check if Vector Tx is satisfied */
2930                 if (fm10k_tx_vec_condition_check(txq))
2931                         use_sse = 0;
2932         }
2933
2934         if (use_sse) {
2935                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2936                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2937                         txq = dev->data->tx_queues[i];
2938                         fm10k_txq_vec_setup(txq);
2939                 }
2940                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2941                 dev->tx_pkt_prepare = NULL;
2942         } else {
2943                 dev->tx_pkt_burst = fm10k_xmit_pkts;
2944                 dev->tx_pkt_prepare = fm10k_prep_pkts;
2945                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2946         }
2947 }
2948
2949 static void __attribute__((cold))
2950 fm10k_set_rx_function(struct rte_eth_dev *dev)
2951 {
2952         struct fm10k_dev_info *dev_info =
2953                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2954         uint16_t i, rx_using_sse;
2955         uint16_t rx_ftag_en = 0;
2956
2957         if (fm10k_check_ftag(dev->device->devargs))
2958                 rx_ftag_en = 1;
2959
2960         /* In order to allow Vector Rx there are a few configuration
2961          * conditions to be met.
2962          */
2963         if (!fm10k_rx_vec_condition_check(dev) &&
2964                         dev_info->rx_vec_allowed && !rx_ftag_en) {
2965                 if (dev->data->scattered_rx)
2966                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
2967                 else
2968                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
2969         } else if (dev->data->scattered_rx)
2970                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
2971         else
2972                 dev->rx_pkt_burst = fm10k_recv_pkts;
2973
2974         rx_using_sse =
2975                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
2976                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
2977
2978         if (rx_using_sse)
2979                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
2980         else
2981                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
2982
2983         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2984                 return;
2985
2986         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2987                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
2988
2989                 rxq->rx_using_sse = rx_using_sse;
2990                 rxq->rx_ftag_en = rx_ftag_en;
2991         }
2992 }
2993
2994 static void
2995 fm10k_params_init(struct rte_eth_dev *dev)
2996 {
2997         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2998         struct fm10k_dev_info *info =
2999                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
3000
3001         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
3002          * there is no way to get link status without reading BAR4.  Until this
3003          * works, assume we have maximum bandwidth.
3004          * @todo - fix bus info
3005          */
3006         hw->bus_caps.speed = fm10k_bus_speed_8000;
3007         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
3008         hw->bus_caps.payload = fm10k_bus_payload_512;
3009         hw->bus.speed = fm10k_bus_speed_8000;
3010         hw->bus.width = fm10k_bus_width_pcie_x8;
3011         hw->bus.payload = fm10k_bus_payload_256;
3012
3013         info->rx_vec_allowed = true;
3014 }
3015
3016 static int
3017 eth_fm10k_dev_init(struct rte_eth_dev *dev)
3018 {
3019         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3020         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3021         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3022         int diag, i;
3023         struct fm10k_macvlan_filter_info *macvlan;
3024
3025         PMD_INIT_FUNC_TRACE();
3026
3027         dev->dev_ops = &fm10k_eth_dev_ops;
3028         dev->rx_pkt_burst = &fm10k_recv_pkts;
3029         dev->tx_pkt_burst = &fm10k_xmit_pkts;
3030         dev->tx_pkt_prepare = &fm10k_prep_pkts;
3031
3032         /*
3033          * Primary process does the whole initialization, for secondary
3034          * processes, we just select the same Rx and Tx function as primary.
3035          */
3036         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
3037                 fm10k_set_rx_function(dev);
3038                 fm10k_set_tx_function(dev);
3039                 return 0;
3040         }
3041
3042         rte_eth_copy_pci_info(dev, pdev);
3043
3044         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
3045         memset(macvlan, 0, sizeof(*macvlan));
3046         /* Vendor and Device ID need to be set before init of shared code */
3047         memset(hw, 0, sizeof(*hw));
3048         hw->device_id = pdev->id.device_id;
3049         hw->vendor_id = pdev->id.vendor_id;
3050         hw->subsystem_device_id = pdev->id.subsystem_device_id;
3051         hw->subsystem_vendor_id = pdev->id.subsystem_vendor_id;
3052         hw->revision_id = 0;
3053         hw->hw_addr = (void *)pdev->mem_resource[0].addr;
3054         if (hw->hw_addr == NULL) {
3055                 PMD_INIT_LOG(ERR, "Bad mem resource."
3056                         " Try to blacklist unused devices.");
3057                 return -EIO;
3058         }
3059
3060         /* Store fm10k_adapter pointer */
3061         hw->back = dev->data->dev_private;
3062
3063         /* Initialize the shared code */
3064         diag = fm10k_init_shared_code(hw);
3065         if (diag != FM10K_SUCCESS) {
3066                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
3067                 return -EIO;
3068         }
3069
3070         /* Initialize parameters */
3071         fm10k_params_init(dev);
3072
3073         /* Initialize the hw */
3074         diag = fm10k_init_hw(hw);
3075         if (diag != FM10K_SUCCESS) {
3076                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
3077                 return -EIO;
3078         }
3079
3080         /* Initialize MAC address(es) */
3081         dev->data->mac_addrs = rte_zmalloc("fm10k",
3082                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
3083         if (dev->data->mac_addrs == NULL) {
3084                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
3085                 return -ENOMEM;
3086         }
3087
3088         diag = fm10k_read_mac_addr(hw);
3089
3090         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
3091                         &dev->data->mac_addrs[0]);
3092
3093         if (diag != FM10K_SUCCESS ||
3094                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
3095
3096                 /* Generate a random addr */
3097                 eth_random_addr(hw->mac.addr);
3098                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
3099                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
3100                 &dev->data->mac_addrs[0]);
3101         }
3102
3103         /* Reset the hw statistics */
3104         fm10k_stats_reset(dev);
3105
3106         /* Reset the hw */
3107         diag = fm10k_reset_hw(hw);
3108         if (diag != FM10K_SUCCESS) {
3109                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
3110                 return -EIO;
3111         }
3112
3113         /* Setup mailbox service */
3114         diag = fm10k_setup_mbx_service(hw);
3115         if (diag != FM10K_SUCCESS) {
3116                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
3117                 return -EIO;
3118         }
3119
3120         /*PF/VF has different interrupt handling mechanism */
3121         if (hw->mac.type == fm10k_mac_pf) {
3122                 /* register callback func to eal lib */
3123                 rte_intr_callback_register(intr_handle,
3124                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3125
3126                 /* enable MISC interrupt */
3127                 fm10k_dev_enable_intr_pf(dev);
3128         } else { /* VF */
3129                 rte_intr_callback_register(intr_handle,
3130                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3131
3132                 fm10k_dev_enable_intr_vf(dev);
3133         }
3134
3135         /* Enable intr after callback registered */
3136         rte_intr_enable(intr_handle);
3137
3138         hw->mac.ops.update_int_moderator(hw);
3139
3140         /* Make sure Switch Manager is ready before going forward. */
3141         if (hw->mac.type == fm10k_mac_pf) {
3142                 int switch_ready = 0;
3143
3144                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3145                         fm10k_mbx_lock(hw);
3146                         hw->mac.ops.get_host_state(hw, &switch_ready);
3147                         fm10k_mbx_unlock(hw);
3148                         if (switch_ready)
3149                                 break;
3150                         /* Delay some time to acquire async LPORT_MAP info. */
3151                         rte_delay_us(WAIT_SWITCH_MSG_US);
3152                 }
3153
3154                 if (switch_ready == 0) {
3155                         PMD_INIT_LOG(ERR, "switch is not ready");
3156                         return -1;
3157                 }
3158         }
3159
3160         /*
3161          * Below function will trigger operations on mailbox, acquire lock to
3162          * avoid race condition from interrupt handler. Operations on mailbox
3163          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
3164          * will handle and generate an interrupt to our side. Then,  FIFO in
3165          * mailbox will be touched.
3166          */
3167         fm10k_mbx_lock(hw);
3168         /* Enable port first */
3169         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
3170                                         MAX_LPORT_NUM, 1);
3171
3172         /* Set unicast mode by default. App can change to other mode in other
3173          * API func.
3174          */
3175         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
3176                                         FM10K_XCAST_MODE_NONE);
3177
3178         fm10k_mbx_unlock(hw);
3179
3180         /* Make sure default VID is ready before going forward. */
3181         if (hw->mac.type == fm10k_mac_pf) {
3182                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3183                         if (hw->mac.default_vid)
3184                                 break;
3185                         /* Delay some time to acquire async port VLAN info. */
3186                         rte_delay_us(WAIT_SWITCH_MSG_US);
3187                 }
3188
3189                 if (!hw->mac.default_vid) {
3190                         PMD_INIT_LOG(ERR, "default VID is not ready");
3191                         return -1;
3192                 }
3193         }
3194
3195         /* Add default mac address */
3196         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
3197                 MAIN_VSI_POOL_NUMBER);
3198
3199         return 0;
3200 }
3201
3202 static int
3203 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
3204 {
3205         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3206         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3207         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3208         PMD_INIT_FUNC_TRACE();
3209
3210         /* only uninitialize in the primary process */
3211         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3212                 return 0;
3213
3214         /* safe to close dev here */
3215         fm10k_dev_close(dev);
3216
3217         dev->dev_ops = NULL;
3218         dev->rx_pkt_burst = NULL;
3219         dev->tx_pkt_burst = NULL;
3220
3221         /* disable uio/vfio intr */
3222         rte_intr_disable(intr_handle);
3223
3224         /*PF/VF has different interrupt handling mechanism */
3225         if (hw->mac.type == fm10k_mac_pf) {
3226                 /* disable interrupt */
3227                 fm10k_dev_disable_intr_pf(dev);
3228
3229                 /* unregister callback func to eal lib */
3230                 rte_intr_callback_unregister(intr_handle,
3231                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3232         } else {
3233                 /* disable interrupt */
3234                 fm10k_dev_disable_intr_vf(dev);
3235
3236                 rte_intr_callback_unregister(intr_handle,
3237                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3238         }
3239
3240         /* free mac memory */
3241         if (dev->data->mac_addrs) {
3242                 rte_free(dev->data->mac_addrs);
3243                 dev->data->mac_addrs = NULL;
3244         }
3245
3246         memset(hw, 0, sizeof(*hw));
3247
3248         return 0;
3249 }
3250
3251 static int eth_fm10k_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
3252         struct rte_pci_device *pci_dev)
3253 {
3254         return rte_eth_dev_pci_generic_probe(pci_dev,
3255                 sizeof(struct fm10k_adapter), eth_fm10k_dev_init);
3256 }
3257
3258 static int eth_fm10k_pci_remove(struct rte_pci_device *pci_dev)
3259 {
3260         return rte_eth_dev_pci_generic_remove(pci_dev, eth_fm10k_dev_uninit);
3261 }
3262
3263 /*
3264  * The set of PCI devices this driver supports. This driver will enable both PF
3265  * and SRIOV-VF devices.
3266  */
3267 static const struct rte_pci_id pci_id_fm10k_map[] = {
3268         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_PF) },
3269         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_SDI_FM10420_QDA2) },
3270         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_VF) },
3271         { .vendor_id = 0, /* sentinel */ },
3272 };
3273
3274 static struct rte_pci_driver rte_pmd_fm10k = {
3275         .id_table = pci_id_fm10k_map,
3276         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
3277                      RTE_PCI_DRV_IOVA_AS_VA,
3278         .probe = eth_fm10k_pci_probe,
3279         .remove = eth_fm10k_pci_remove,
3280 };
3281
3282 RTE_PMD_REGISTER_PCI(net_fm10k, rte_pmd_fm10k);
3283 RTE_PMD_REGISTER_PCI_TABLE(net_fm10k, pci_id_fm10k_map);
3284 RTE_PMD_REGISTER_KMOD_DEP(net_fm10k, "* igb_uio | uio_pci_generic | vfio-pci");
3285
3286 RTE_INIT(fm10k_init_log)
3287 {
3288         fm10k_logtype_init = rte_log_register("pmd.net.fm10k.init");
3289         if (fm10k_logtype_init >= 0)
3290                 rte_log_set_level(fm10k_logtype_init, RTE_LOG_NOTICE);
3291         fm10k_logtype_driver = rte_log_register("pmd.net.fm10k.driver");
3292         if (fm10k_logtype_driver >= 0)
3293                 rte_log_set_level(fm10k_logtype_driver, RTE_LOG_NOTICE);
3294 }