drivers/net/fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2016 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_ethdev_pci.h>
  36 #include <rte_malloc.h>
  37 #include <rte_memzone.h>
  38 #include <rte_string_fns.h>
  39 #include <rte_dev.h>
  40 #include <rte_spinlock.h>
  41 #include <rte_kvargs.h>
  42
  43 #include "fm10k.h"
  44 #include "base/fm10k_api.h"
  45
  46 /* Default delay to acquire mailbox lock */
  47 #define FM10K_MBXLOCK_DELAY_US 20
  48 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  49
  50 #define MAIN_VSI_POOL_NUMBER 0
  51
  52 /* Max try times to acquire switch status */
  53 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  54 /* Wait interval to get switch status */
  55 #define WAIT_SWITCH_MSG_US    100000
  56 /* A period of quiescence for switch */
  57 #define FM10K_SWITCH_QUIESCE_US 10000
  58 /* Number of chars per uint32 type */
  59 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  60 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  61
  62 /* default 1:1 map from queue ID to interrupt vector ID */
  63 #define Q2V(pci_dev, queue_id) ((pci_dev)->intr_handle.intr_vec[queue_id])
  64
  65 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  66 #define MAX_LPORT_NUM    128
  67 #define GLORT_FD_Q_BASE  0x40
  68 #define GLORT_PF_MASK    0xFFC0
  69 #define GLORT_FD_MASK    GLORT_PF_MASK
  70 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  71
  72 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  73 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  74 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  75 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  76 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  77 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  78 static int
  79 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  80 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  81         const u8 *mac, bool add, uint32_t pool);
  82 static void fm10k_tx_queue_release(void *queue);
  83 static void fm10k_rx_queue_release(void *queue);
  84 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  85 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  86 static int fm10k_check_ftag(struct rte_devargs *devargs);
  87 static int fm10k_link_update(struct rte_eth_dev *dev, int wait_to_complete);
  88
  89 struct fm10k_xstats_name_off {
  90         char name[RTE_ETH_XSTATS_NAME_SIZE];
  91         unsigned offset;
  92 };
  93
  94 struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  95         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  96         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  97         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  98         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  99         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
 100         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
 101         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
 102         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
 103                 nodesc_drop)},
 104 };
 105
 106 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
 107                 sizeof(fm10k_hw_stats_strings[0]))
 108
 109 struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
 110         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
 111         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
 112         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
 113 };
 114
 115 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
 116                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
 117
 118 struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
 119         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 120         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 121 };
 122
 123 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 124                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 125
 126 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 127                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 128 static int
 129 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 130
 131 static void
 132 fm10k_mbx_initlock(struct fm10k_hw *hw)
 133 {
 134         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 135 }
 136
 137 static void
 138 fm10k_mbx_lock(struct fm10k_hw *hw)
 139 {
 140         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 141                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 142 }
 143
 144 static void
 145 fm10k_mbx_unlock(struct fm10k_hw *hw)
 146 {
 147         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 148 }
 149
 150 /* Stubs needed for linkage when vPMD is disabled */
 151 int __attribute__((weak))
 152 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 153 {
 154         return -1;
 155 }
 156
 157 uint16_t __attribute__((weak))
 158 fm10k_recv_pkts_vec(
 159         __rte_unused void *rx_queue,
 160         __rte_unused struct rte_mbuf **rx_pkts,
 161         __rte_unused uint16_t nb_pkts)
 162 {
 163         return 0;
 164 }
 165
 166 uint16_t __attribute__((weak))
 167 fm10k_recv_scattered_pkts_vec(
 168                 __rte_unused void *rx_queue,
 169                 __rte_unused struct rte_mbuf **rx_pkts,
 170                 __rte_unused uint16_t nb_pkts)
 171 {
 172         return 0;
 173 }
 174
 175 int __attribute__((weak))
 176 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 177
 178 {
 179         return -1;
 180 }
 181
 182 void __attribute__((weak))
 183 fm10k_rx_queue_release_mbufs_vec(
 184                 __rte_unused struct fm10k_rx_queue *rxq)
 185 {
 186         return;
 187 }
 188
 189 void __attribute__((weak))
 190 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 191 {
 192         return;
 193 }
 194
 195 int __attribute__((weak))
 196 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 197 {
 198         return -1;
 199 }
 200
 201 uint16_t __attribute__((weak))
 202 fm10k_xmit_fixed_burst_vec(__rte_unused void *tx_queue,
 203                            __rte_unused struct rte_mbuf **tx_pkts,
 204                            __rte_unused uint16_t nb_pkts)
 205 {
 206         return 0;
 207 }
 208
 209 /*
 210  * reset queue to initial state, allocate software buffers used when starting
 211  * device.
 212  * return 0 on success
 213  * return -ENOMEM if buffers cannot be allocated
 214  * return -EINVAL if buffers do not satisfy alignment condition
 215  */
 216 static inline int
 217 rx_queue_reset(struct fm10k_rx_queue *q)
 218 {
 219         static const union fm10k_rx_desc zero = {{0} };
 220         uint64_t dma_addr;
 221         int i, diag;
 222         PMD_INIT_FUNC_TRACE();
 223
 224         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 225         if (diag != 0)
 226                 return -ENOMEM;
 227
 228         for (i = 0; i < q->nb_desc; ++i) {
 229                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 230                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 231                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 232                                                 q->nb_desc);
 233                         return -EINVAL;
 234                 }
 235                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 236                 q->hw_ring[i].q.pkt_addr = dma_addr;
 237                 q->hw_ring[i].q.hdr_addr = dma_addr;
 238         }
 239
 240         /* initialize extra software ring entries. Space for these extra
 241          * entries is always allocated.
 242          */
 243         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 244         for (i = 0; i < q->nb_fake_desc; ++i) {
 245                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 246                 q->hw_ring[q->nb_desc + i] = zero;
 247         }
 248
 249         q->next_dd = 0;
 250         q->next_alloc = 0;
 251         q->next_trigger = q->alloc_thresh - 1;
 252         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 253         q->rxrearm_start = 0;
 254         q->rxrearm_nb = 0;
 255
 256         return 0;
 257 }
 258
 259 /*
 260  * clean queue, descriptor rings, free software buffers used when stopping
 261  * device.
 262  */
 263 static inline void
 264 rx_queue_clean(struct fm10k_rx_queue *q)
 265 {
 266         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 267         uint32_t i;
 268         PMD_INIT_FUNC_TRACE();
 269
 270         /* zero descriptor rings */
 271         for (i = 0; i < q->nb_desc; ++i)
 272                 q->hw_ring[i] = zero;
 273
 274         /* zero faked descriptors */
 275         for (i = 0; i < q->nb_fake_desc; ++i)
 276                 q->hw_ring[q->nb_desc + i] = zero;
 277
 278         /* vPMD driver has a different way of releasing mbufs. */
 279         if (q->rx_using_sse) {
 280                 fm10k_rx_queue_release_mbufs_vec(q);
 281                 return;
 282         }
 283
 284         /* free software buffers */
 285         for (i = 0; i < q->nb_desc; ++i) {
 286                 if (q->sw_ring[i]) {
 287                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 288                         q->sw_ring[i] = NULL;
 289                 }
 290         }
 291 }
 292
 293 /*
 294  * free all queue memory used when releasing the queue (i.e. configure)
 295  */
 296 static inline void
 297 rx_queue_free(struct fm10k_rx_queue *q)
 298 {
 299         PMD_INIT_FUNC_TRACE();
 300         if (q) {
 301                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 302                 rx_queue_clean(q);
 303                 if (q->sw_ring) {
 304                         rte_free(q->sw_ring);
 305                         q->sw_ring = NULL;
 306                 }
 307                 rte_free(q);
 308                 q = NULL;
 309         }
 310 }
 311
 312 /*
 313  * disable RX queue, wait unitl HW finished necessary flush operation
 314  */
 315 static inline int
 316 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 317 {
 318         uint32_t reg, i;
 319
 320         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 321         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 322                         reg & ~FM10K_RXQCTL_ENABLE);
 323
 324         /* Wait 100us at most */
 325         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 326                 rte_delay_us(1);
 327                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 328                 if (!(reg & FM10K_RXQCTL_ENABLE))
 329                         break;
 330         }
 331
 332         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 333                 return -1;
 334
 335         return 0;
 336 }
 337
 338 /*
 339  * reset queue to initial state, allocate software buffers used when starting
 340  * device
 341  */
 342 static inline void
 343 tx_queue_reset(struct fm10k_tx_queue *q)
 344 {
 345         PMD_INIT_FUNC_TRACE();
 346         q->last_free = 0;
 347         q->next_free = 0;
 348         q->nb_used = 0;
 349         q->nb_free = q->nb_desc - 1;
 350         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 351         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 352 }
 353
 354 /*
 355  * clean queue, descriptor rings, free software buffers used when stopping
 356  * device
 357  */
 358 static inline void
 359 tx_queue_clean(struct fm10k_tx_queue *q)
 360 {
 361         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 362         uint32_t i;
 363         PMD_INIT_FUNC_TRACE();
 364
 365         /* zero descriptor rings */
 366         for (i = 0; i < q->nb_desc; ++i)
 367                 q->hw_ring[i] = zero;
 368
 369         /* free software buffers */
 370         for (i = 0; i < q->nb_desc; ++i) {
 371                 if (q->sw_ring[i]) {
 372                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 373                         q->sw_ring[i] = NULL;
 374                 }
 375         }
 376 }
 377
 378 /*
 379  * free all queue memory used when releasing the queue (i.e. configure)
 380  */
 381 static inline void
 382 tx_queue_free(struct fm10k_tx_queue *q)
 383 {
 384         PMD_INIT_FUNC_TRACE();
 385         if (q) {
 386                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 387                 tx_queue_clean(q);
 388                 if (q->rs_tracker.list) {
 389                         rte_free(q->rs_tracker.list);
 390                         q->rs_tracker.list = NULL;
 391                 }
 392                 if (q->sw_ring) {
 393                         rte_free(q->sw_ring);
 394                         q->sw_ring = NULL;
 395                 }
 396                 rte_free(q);
 397                 q = NULL;
 398         }
 399 }
 400
 401 /*
 402  * disable TX queue, wait unitl HW finished necessary flush operation
 403  */
 404 static inline int
 405 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 406 {
 407         uint32_t reg, i;
 408
 409         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 410         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 411                         reg & ~FM10K_TXDCTL_ENABLE);
 412
 413         /* Wait 100us at most */
 414         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 415                 rte_delay_us(1);
 416                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 417                 if (!(reg & FM10K_TXDCTL_ENABLE))
 418                         break;
 419         }
 420
 421         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 422                 return -1;
 423
 424         return 0;
 425 }
 426
 427 static int
 428 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 429 {
 430         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 431         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 432         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 433         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 434
 435         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 436
 437         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 438                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 439                 return -EINVAL;
 440         }
 441
 442         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 443                 return 0;
 444
 445         if (hw->mac.type == fm10k_mac_vf) {
 446                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 447                 return -EINVAL;
 448         }
 449
 450         /* Check VMDQ queue pool number */
 451         if (vmdq_conf->nb_queue_pools >
 452                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 453                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 454                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 455                         vmdq_conf->nb_queue_pools);
 456                 return -EINVAL;
 457         }
 458
 459         return 0;
 460 }
 461
 462 static const struct fm10k_txq_ops def_txq_ops = {
 463         .reset = tx_queue_reset,
 464 };
 465
 466 static int
 467 fm10k_dev_configure(struct rte_eth_dev *dev)
 468 {
 469         int ret;
 470
 471         PMD_INIT_FUNC_TRACE();
 472
 473         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 474                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 475         /* multipe queue mode checking */
 476         ret  = fm10k_check_mq_mode(dev);
 477         if (ret != 0) {
 478                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 479                             ret);
 480                 return ret;
 481         }
 482
 483         return 0;
 484 }
 485
 486 /* fls = find last set bit = 32 minus the number of leading zeros */
 487 #ifndef fls
 488 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 489 #endif
 490
 491 static void
 492 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 493 {
 494         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 495         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 496         uint32_t i;
 497
 498         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 499
 500         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 501                 if (!vmdq_conf->pool_map[i].pools)
 502                         continue;
 503                 fm10k_mbx_lock(hw);
 504                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 505                 fm10k_mbx_unlock(hw);
 506         }
 507 }
 508
 509 static void
 510 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 511 {
 512         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 513
 514         /* Add default mac address */
 515         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 516                 MAIN_VSI_POOL_NUMBER);
 517 }
 518
 519 static void
 520 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 521 {
 522         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 523         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 524         uint32_t mrqc, *key, i, reta, j;
 525         uint64_t hf;
 526
 527 #define RSS_KEY_SIZE 40
 528         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 529                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 530                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 531                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 532                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 533                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 534         };
 535
 536         if (dev->data->nb_rx_queues == 1 ||
 537             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 538             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0) {
 539                 FM10K_WRITE_REG(hw, FM10K_MRQC(0), 0);
 540                 return;
 541         }
 542
 543         /* random key is rss_intel_key (default) or user provided (rss_key) */
 544         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 545                 key = (uint32_t *)rss_intel_key;
 546         else
 547                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 548
 549         /* Now fill our hash function seeds, 4 bytes at a time */
 550         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 551                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 552
 553         /*
 554          * Fill in redirection table
 555          * The byte-swap is needed because NIC registers are in
 556          * little-endian order.
 557          */
 558         reta = 0;
 559         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 560                 if (j == dev->data->nb_rx_queues)
 561                         j = 0;
 562                 reta = (reta << CHAR_BIT) | j;
 563                 if ((i & 3) == 3)
 564                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 565                                         rte_bswap32(reta));
 566         }
 567
 568         /*
 569          * Generate RSS hash based on packet types, TCP/UDP
 570          * port numbers and/or IPv4/v6 src and dst addresses
 571          */
 572         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 573         mrqc = 0;
 574         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 575         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 576         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 577         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 578         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 579         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 580         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 581         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 582         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 583
 584         if (mrqc == 0) {
 585                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 586                         "supported", hf);
 587                 return;
 588         }
 589
 590         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 591 }
 592
 593 static void
 594 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 595 {
 596         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 597         uint32_t i;
 598
 599         for (i = 0; i < nb_lport_new; i++) {
 600                 /* Set unicast mode by default. App can change
 601                  * to other mode in other API func.
 602                  */
 603                 fm10k_mbx_lock(hw);
 604                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 605                         FM10K_XCAST_MODE_NONE);
 606                 fm10k_mbx_unlock(hw);
 607         }
 608 }
 609
 610 static void
 611 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 612 {
 613         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 614         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 615         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 616         struct fm10k_macvlan_filter_info *macvlan;
 617         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 618         uint16_t nb_lport_new;
 619
 620         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 621         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 622
 623         fm10k_dev_rss_configure(dev);
 624
 625         /* only PF supports VMDQ */
 626         if (hw->mac.type != fm10k_mac_pf)
 627                 return;
 628
 629         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 630                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 631
 632         /* no pool number change, no need to update logic port and VLAN/MAC */
 633         if (macvlan->nb_queue_pools == nb_queue_pools)
 634                 return;
 635
 636         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 637         fm10k_dev_logic_port_update(dev, nb_lport_new);
 638
 639         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 640         memset(dev->data->mac_addrs, 0,
 641                 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 642         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
 643                 &dev->data->mac_addrs[0]);
 644         memset(macvlan, 0, sizeof(*macvlan));
 645         macvlan->nb_queue_pools = nb_queue_pools;
 646
 647         if (nb_queue_pools)
 648                 fm10k_dev_vmdq_rx_configure(dev);
 649         else
 650                 fm10k_dev_pf_main_vsi_reset(dev);
 651 }
 652
 653 static int
 654 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 655 {
 656         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 657         int i, ret;
 658         struct fm10k_tx_queue *txq;
 659         uint64_t base_addr;
 660         uint32_t size;
 661
 662         /* Disable TXINT to avoid possible interrupt */
 663         for (i = 0; i < hw->mac.max_queues; i++)
 664                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 665                                 3 << FM10K_TXINT_TIMER_SHIFT);
 666
 667         /* Setup TX queue */
 668         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 669                 txq = dev->data->tx_queues[i];
 670                 base_addr = txq->hw_ring_phys_addr;
 671                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 672
 673                 /* disable queue to avoid issues while updating state */
 674                 ret = tx_queue_disable(hw, i);
 675                 if (ret) {
 676                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 677                         return -1;
 678                 }
 679                 /* Enable use of FTAG bit in TX descriptor, PFVTCTL
 680                  * register is read-only for VF.
 681                  */
 682                 if (fm10k_check_ftag(dev->device->devargs)) {
 683                         if (hw->mac.type == fm10k_mac_pf) {
 684                                 FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i),
 685                                                 FM10K_PFVTCTL_FTAG_DESC_ENABLE);
 686                                 PMD_INIT_LOG(DEBUG, "FTAG mode is enabled");
 687                         } else {
 688                                 PMD_INIT_LOG(ERR, "VF FTAG is not supported.");
 689                                 return -ENOTSUP;
 690                         }
 691                 }
 692
 693                 /* set location and size for descriptor ring */
 694                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 695                                 base_addr & UINT64_LOWER_32BITS_MASK);
 696                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 697                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 698                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 699
 700                 /* assign default SGLORT for each TX queue by PF */
 701                 if (hw->mac.type == fm10k_mac_pf)
 702                         FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map);
 703         }
 704
 705         /* set up vector or scalar TX function as appropriate */
 706         fm10k_set_tx_function(dev);
 707
 708         return 0;
 709 }
 710
 711 static int
 712 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 713 {
 714         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 715         struct fm10k_macvlan_filter_info *macvlan;
 716         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
 717         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
 718         int i, ret;
 719         struct fm10k_rx_queue *rxq;
 720         uint64_t base_addr;
 721         uint32_t size;
 722         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 723         uint32_t logic_port = hw->mac.dglort_map;
 724         uint16_t buf_size;
 725         uint16_t queue_stride = 0;
 726
 727         /* enable RXINT for interrupt mode */
 728         i = 0;
 729         if (rte_intr_dp_is_en(intr_handle)) {
 730                 for (; i < dev->data->nb_rx_queues; i++) {
 731                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(pdev, i));
 732                         if (hw->mac.type == fm10k_mac_pf)
 733                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
 734                                         FM10K_ITR_AUTOMASK |
 735                                         FM10K_ITR_MASK_CLEAR);
 736                         else
 737                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
 738                                         FM10K_ITR_AUTOMASK |
 739                                         FM10K_ITR_MASK_CLEAR);
 740                 }
 741         }
 742         /* Disable other RXINT to avoid possible interrupt */
 743         for (; i < hw->mac.max_queues; i++)
 744                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 745                         3 << FM10K_RXINT_TIMER_SHIFT);
 746
 747         /* Setup RX queues */
 748         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 749                 rxq = dev->data->rx_queues[i];
 750                 base_addr = rxq->hw_ring_phys_addr;
 751                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 752
 753                 /* disable queue to avoid issues while updating state */
 754                 ret = rx_queue_disable(hw, i);
 755                 if (ret) {
 756                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 757                         return -1;
 758                 }
 759
 760                 /* Setup the Base and Length of the Rx Descriptor Ring */
 761                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 762                                 base_addr & UINT64_LOWER_32BITS_MASK);
 763                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 764                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 765                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 766
 767                 /* Configure the Rx buffer size for one buff without split */
 768                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 769                         RTE_PKTMBUF_HEADROOM);
 770                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 771                  * reserved for this purpose, and the worst case could be 511B.
 772                  * But SRR reg assumes all buffers have the same size. In order
 773                  * to fill the gap, we'll have to consider the worst case and
 774                  * assume 512B is reserved. If we don't do so, it's possible
 775                  * for HW to overwrite data to next mbuf.
 776                  */
 777                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 778
 779                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 780                                 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
 781                                 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
 782
 783                 /* It adds dual VLAN length for supporting dual VLAN */
 784                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 785                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 786                         dev->data->dev_conf.rxmode.enable_scatter) {
 787                         uint32_t reg;
 788                         dev->data->scattered_rx = 1;
 789                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 790                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 791                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 792                 }
 793
 794                 /* Enable drop on empty, it's RO for VF */
 795                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 796                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 797
 798                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 799                 FM10K_WRITE_FLUSH(hw);
 800         }
 801
 802         /* Configure VMDQ/RSS if applicable */
 803         fm10k_dev_mq_rx_configure(dev);
 804
 805         /* Decide the best RX function */
 806         fm10k_set_rx_function(dev);
 807
 808         /* update RX_SGLORT for loopback suppress*/
 809         if (hw->mac.type != fm10k_mac_pf)
 810                 return 0;
 811         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 812         if (macvlan->nb_queue_pools)
 813                 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
 814         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 815                 if (i && queue_stride && !(i % queue_stride))
 816                         logic_port++;
 817                 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
 818         }
 819
 820         return 0;
 821 }
 822
 823 static int
 824 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 825 {
 826         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 827         int err = -1;
 828         uint32_t reg;
 829         struct fm10k_rx_queue *rxq;
 830
 831         PMD_INIT_FUNC_TRACE();
 832
 833         if (rx_queue_id < dev->data->nb_rx_queues) {
 834                 rxq = dev->data->rx_queues[rx_queue_id];
 835                 err = rx_queue_reset(rxq);
 836                 if (err == -ENOMEM) {
 837                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 838                         return err;
 839                 } else if (err == -EINVAL) {
 840                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 841                                 " %d", err);
 842                         return err;
 843                 }
 844
 845                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 846                  * Note: this must be done AFTER the queue is enabled on real
 847                  * hardware, but BEFORE the queue is enabled when using the
 848                  * emulation platform. Do it in both places for now and remove
 849                  * this comment and the following two register writes when the
 850                  * emulation platform is no longer being used.
 851                  */
 852                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 853                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 854
 855                 /* Set PF ownership flag for PF devices */
 856                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 857                 if (hw->mac.type == fm10k_mac_pf)
 858                         reg |= FM10K_RXQCTL_PF;
 859                 reg |= FM10K_RXQCTL_ENABLE;
 860                 /* enable RX queue */
 861                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 862                 FM10K_WRITE_FLUSH(hw);
 863
 864                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 865                  * Note: this must be done AFTER the queue is enabled
 866                  */
 867                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 868                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 869                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 870         }
 871
 872         return err;
 873 }
 874
 875 static int
 876 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 877 {
 878         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 879
 880         PMD_INIT_FUNC_TRACE();
 881
 882         if (rx_queue_id < dev->data->nb_rx_queues) {
 883                 /* Disable RX queue */
 884                 rx_queue_disable(hw, rx_queue_id);
 885
 886                 /* Free mbuf and clean HW ring */
 887                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 888                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 889         }
 890
 891         return 0;
 892 }
 893
 894 static int
 895 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 896 {
 897         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 898         /** @todo - this should be defined in the shared code */
 899 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 900         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 901         int err = 0;
 902
 903         PMD_INIT_FUNC_TRACE();
 904
 905         if (tx_queue_id < dev->data->nb_tx_queues) {
 906                 struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 907
 908                 q->ops->reset(q);
 909
 910                 /* reset head and tail pointers */
 911                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 912                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 913
 914                 /* enable TX queue */
 915                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 916                                         FM10K_TXDCTL_ENABLE | txdctl);
 917                 FM10K_WRITE_FLUSH(hw);
 918                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 919         } else
 920                 err = -1;
 921
 922         return err;
 923 }
 924
 925 static int
 926 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 927 {
 928         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 929
 930         PMD_INIT_FUNC_TRACE();
 931
 932         if (tx_queue_id < dev->data->nb_tx_queues) {
 933                 tx_queue_disable(hw, tx_queue_id);
 934                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 935                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 936         }
 937
 938         return 0;
 939 }
 940
 941 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 942 {
 943         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 944                 != FM10K_DGLORTMAP_NONE);
 945 }
 946
 947 static void
 948 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 949 {
 950         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 951         int status;
 952
 953         PMD_INIT_FUNC_TRACE();
 954
 955         /* Return if it didn't acquire valid glort range */
 956         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 957                 return;
 958
 959         fm10k_mbx_lock(hw);
 960         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 961                                 FM10K_XCAST_MODE_PROMISC);
 962         fm10k_mbx_unlock(hw);
 963
 964         if (status != FM10K_SUCCESS)
 965                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 966 }
 967
 968 static void
 969 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 970 {
 971         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 972         uint8_t mode;
 973         int status;
 974
 975         PMD_INIT_FUNC_TRACE();
 976
 977         /* Return if it didn't acquire valid glort range */
 978         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 979                 return;
 980
 981         if (dev->data->all_multicast == 1)
 982                 mode = FM10K_XCAST_MODE_ALLMULTI;
 983         else
 984                 mode = FM10K_XCAST_MODE_NONE;
 985
 986         fm10k_mbx_lock(hw);
 987         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 988                                 mode);
 989         fm10k_mbx_unlock(hw);
 990
 991         if (status != FM10K_SUCCESS)
 992                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 993 }
 994
 995 static void
 996 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 997 {
 998         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 999         int status;
1000
1001         PMD_INIT_FUNC_TRACE();
1002
1003         /* Return if it didn't acquire valid glort range */
1004         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1005                 return;
1006
1007         /* If promiscuous mode is enabled, it doesn't make sense to enable
1008          * allmulticast and disable promiscuous since fm10k only can select
1009          * one of the modes.
1010          */
1011         if (dev->data->promiscuous) {
1012                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
1013                         "needn't enable allmulticast");
1014                 return;
1015         }
1016
1017         fm10k_mbx_lock(hw);
1018         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1019                                 FM10K_XCAST_MODE_ALLMULTI);
1020         fm10k_mbx_unlock(hw);
1021
1022         if (status != FM10K_SUCCESS)
1023                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
1024 }
1025
1026 static void
1027 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
1028 {
1029         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1030         int status;
1031
1032         PMD_INIT_FUNC_TRACE();
1033
1034         /* Return if it didn't acquire valid glort range */
1035         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1036                 return;
1037
1038         if (dev->data->promiscuous) {
1039                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1040                         "since promisc mode is enabled");
1041                 return;
1042         }
1043
1044         fm10k_mbx_lock(hw);
1045         /* Change mode to unicast mode */
1046         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1047                                 FM10K_XCAST_MODE_NONE);
1048         fm10k_mbx_unlock(hw);
1049
1050         if (status != FM10K_SUCCESS)
1051                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1052 }
1053
1054 static void
1055 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1056 {
1057         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1058         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1059         uint16_t nb_queue_pools;
1060         struct fm10k_macvlan_filter_info *macvlan;
1061
1062         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1063         nb_queue_pools = macvlan->nb_queue_pools;
1064         pool_len = nb_queue_pools ? fls(nb_queue_pools - 1) : 0;
1065         rss_len = fls(dev->data->nb_rx_queues - 1) - pool_len;
1066
1067         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1068         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1069         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1070                         hw->mac.dglort_map;
1071         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1072         /* Configure VMDQ/RSS DGlort Decoder */
1073         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1074
1075         /* Flow Director configurations, only queue number is valid. */
1076         dglortdec = fls(dev->data->nb_rx_queues - 1);
1077         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1078                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1079         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1080         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1081
1082         /* Invalidate all other GLORT entries */
1083         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1084                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1085                                 FM10K_DGLORTMAP_NONE);
1086 }
1087
1088 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1089 static int
1090 fm10k_dev_start(struct rte_eth_dev *dev)
1091 {
1092         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1093         int i, diag;
1094
1095         PMD_INIT_FUNC_TRACE();
1096
1097         /* stop, init, then start the hw */
1098         diag = fm10k_stop_hw(hw);
1099         if (diag != FM10K_SUCCESS) {
1100                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1101                 return -EIO;
1102         }
1103
1104         diag = fm10k_init_hw(hw);
1105         if (diag != FM10K_SUCCESS) {
1106                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1107                 return -EIO;
1108         }
1109
1110         diag = fm10k_start_hw(hw);
1111         if (diag != FM10K_SUCCESS) {
1112                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1113                 return -EIO;
1114         }
1115
1116         diag = fm10k_dev_tx_init(dev);
1117         if (diag) {
1118                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1119                 return diag;
1120         }
1121
1122         if (fm10k_dev_rxq_interrupt_setup(dev))
1123                 return -EIO;
1124
1125         diag = fm10k_dev_rx_init(dev);
1126         if (diag) {
1127                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1128                 return diag;
1129         }
1130
1131         if (hw->mac.type == fm10k_mac_pf)
1132                 fm10k_dev_dglort_map_configure(dev);
1133
1134         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1135                 struct fm10k_rx_queue *rxq;
1136                 rxq = dev->data->rx_queues[i];
1137
1138                 if (rxq->rx_deferred_start)
1139                         continue;
1140                 diag = fm10k_dev_rx_queue_start(dev, i);
1141                 if (diag != 0) {
1142                         int j;
1143                         for (j = 0; j < i; ++j)
1144                                 rx_queue_clean(dev->data->rx_queues[j]);
1145                         return diag;
1146                 }
1147         }
1148
1149         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1150                 struct fm10k_tx_queue *txq;
1151                 txq = dev->data->tx_queues[i];
1152
1153                 if (txq->tx_deferred_start)
1154                         continue;
1155                 diag = fm10k_dev_tx_queue_start(dev, i);
1156                 if (diag != 0) {
1157                         int j;
1158                         for (j = 0; j < i; ++j)
1159                                 tx_queue_clean(dev->data->tx_queues[j]);
1160                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1161                                 rx_queue_clean(dev->data->rx_queues[j]);
1162                         return diag;
1163                 }
1164         }
1165
1166         /* Update default vlan when not in VMDQ mode */
1167         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1168                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1169
1170         fm10k_link_update(dev, 0);
1171
1172         return 0;
1173 }
1174
1175 static void
1176 fm10k_dev_stop(struct rte_eth_dev *dev)
1177 {
1178         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1179         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1180         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
1181         int i;
1182
1183         PMD_INIT_FUNC_TRACE();
1184
1185         if (dev->data->tx_queues)
1186                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1187                         fm10k_dev_tx_queue_stop(dev, i);
1188
1189         if (dev->data->rx_queues)
1190                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1191                         fm10k_dev_rx_queue_stop(dev, i);
1192
1193         /* Disable datapath event */
1194         if (rte_intr_dp_is_en(intr_handle)) {
1195                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1196                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1197                                 3 << FM10K_RXINT_TIMER_SHIFT);
1198                         if (hw->mac.type == fm10k_mac_pf)
1199                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
1200                                         FM10K_ITR_MASK_SET);
1201                         else
1202                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
1203                                         FM10K_ITR_MASK_SET);
1204                 }
1205         }
1206         /* Clean datapath event and queue/vec mapping */
1207         rte_intr_efd_disable(intr_handle);
1208         rte_free(intr_handle->intr_vec);
1209         intr_handle->intr_vec = NULL;
1210 }
1211
1212 static void
1213 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1214 {
1215         int i;
1216
1217         PMD_INIT_FUNC_TRACE();
1218
1219         if (dev->data->tx_queues) {
1220                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1221                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1222
1223                         tx_queue_free(txq);
1224                 }
1225         }
1226
1227         if (dev->data->rx_queues) {
1228                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1229                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1230         }
1231 }
1232
1233 static void
1234 fm10k_dev_close(struct rte_eth_dev *dev)
1235 {
1236         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1237
1238         PMD_INIT_FUNC_TRACE();
1239
1240         fm10k_mbx_lock(hw);
1241         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1242                 MAX_LPORT_NUM, false);
1243         fm10k_mbx_unlock(hw);
1244
1245         /* allow 10ms for device to quiesce */
1246         rte_delay_us(FM10K_SWITCH_QUIESCE_US);
1247
1248         /* Stop mailbox service first */
1249         fm10k_close_mbx_service(hw);
1250         fm10k_dev_stop(dev);
1251         fm10k_dev_queue_release(dev);
1252         fm10k_stop_hw(hw);
1253 }
1254
1255 static int
1256 fm10k_link_update(struct rte_eth_dev *dev,
1257         __rte_unused int wait_to_complete)
1258 {
1259         PMD_INIT_FUNC_TRACE();
1260
1261         /* The host-interface link is always up.  The speed is ~50Gbps per Gen3
1262          * x8 PCIe interface. For now, we leave the speed undefined since there
1263          * is no 50Gbps Ethernet. */
1264         dev->data->dev_link.link_speed  = 0;
1265         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1266         dev->data->dev_link.link_status = ETH_LINK_UP;
1267
1268         return 0;
1269 }
1270
1271 static int fm10k_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1272         struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit)
1273 {
1274         unsigned i, q;
1275         unsigned count = 0;
1276
1277         if (xstats_names != NULL) {
1278                 /* Note: limit checked in rte_eth_xstats_names() */
1279
1280                 /* Global stats */
1281                 for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1282                         snprintf(xstats_names[count].name,
1283                                 sizeof(xstats_names[count].name),
1284                                 "%s", fm10k_hw_stats_strings[count].name);
1285                         count++;
1286                 }
1287
1288                 /* PF queue stats */
1289                 for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1290                         for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1291                                 snprintf(xstats_names[count].name,
1292                                         sizeof(xstats_names[count].name),
1293                                         "rx_q%u_%s", q,
1294                                         fm10k_hw_stats_rx_q_strings[i].name);
1295                                 count++;
1296                         }
1297                         for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1298                                 snprintf(xstats_names[count].name,
1299                                         sizeof(xstats_names[count].name),
1300                                         "tx_q%u_%s", q,
1301                                         fm10k_hw_stats_tx_q_strings[i].name);
1302                                 count++;
1303                         }
1304                 }
1305         }
1306         return FM10K_NB_XSTATS;
1307 }
1308
1309 static int
1310 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1311                  unsigned n)
1312 {
1313         struct fm10k_hw_stats *hw_stats =
1314                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1315         unsigned i, q, count = 0;
1316
1317         if (n < FM10K_NB_XSTATS)
1318                 return FM10K_NB_XSTATS;
1319
1320         /* Global stats */
1321         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1322                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1323                         fm10k_hw_stats_strings[count].offset);
1324                 xstats[count].id = count;
1325                 count++;
1326         }
1327
1328         /* PF queue stats */
1329         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1330                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1331                         xstats[count].value =
1332                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1333                                 fm10k_hw_stats_rx_q_strings[i].offset);
1334                         xstats[count].id = count;
1335                         count++;
1336                 }
1337                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1338                         xstats[count].value =
1339                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1340                                 fm10k_hw_stats_tx_q_strings[i].offset);
1341                         xstats[count].id = count;
1342                         count++;
1343                 }
1344         }
1345
1346         return FM10K_NB_XSTATS;
1347 }
1348
1349 static void
1350 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1351 {
1352         uint64_t ipackets, opackets, ibytes, obytes;
1353         struct fm10k_hw *hw =
1354                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1355         struct fm10k_hw_stats *hw_stats =
1356                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1357         int i;
1358
1359         PMD_INIT_FUNC_TRACE();
1360
1361         fm10k_update_hw_stats(hw, hw_stats);
1362
1363         ipackets = opackets = ibytes = obytes = 0;
1364         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1365                 (i < hw->mac.max_queues); ++i) {
1366                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1367                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1368                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1369                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1370                 ipackets += stats->q_ipackets[i];
1371                 opackets += stats->q_opackets[i];
1372                 ibytes   += stats->q_ibytes[i];
1373                 obytes   += stats->q_obytes[i];
1374         }
1375         stats->ipackets = ipackets;
1376         stats->opackets = opackets;
1377         stats->ibytes = ibytes;
1378         stats->obytes = obytes;
1379 }
1380
1381 static void
1382 fm10k_stats_reset(struct rte_eth_dev *dev)
1383 {
1384         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1385         struct fm10k_hw_stats *hw_stats =
1386                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1387
1388         PMD_INIT_FUNC_TRACE();
1389
1390         memset(hw_stats, 0, sizeof(*hw_stats));
1391         fm10k_rebind_hw_stats(hw, hw_stats);
1392 }
1393
1394 static void
1395 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1396         struct rte_eth_dev_info *dev_info)
1397 {
1398         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1399         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1400
1401         PMD_INIT_FUNC_TRACE();
1402
1403         dev_info->pci_dev            = pdev;
1404         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1405         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1406         dev_info->max_rx_queues      = hw->mac.max_queues;
1407         dev_info->max_tx_queues      = hw->mac.max_queues;
1408         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1409         dev_info->max_hash_mac_addrs = 0;
1410         dev_info->max_vfs            = pdev->max_vfs;
1411         dev_info->vmdq_pool_base     = 0;
1412         dev_info->vmdq_queue_base    = 0;
1413         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1414         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1415         dev_info->rx_offload_capa =
1416                 DEV_RX_OFFLOAD_VLAN_STRIP |
1417                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1418                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1419                 DEV_RX_OFFLOAD_TCP_CKSUM;
1420         dev_info->tx_offload_capa =
1421                 DEV_TX_OFFLOAD_VLAN_INSERT |
1422                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1423                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1424                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1425                 DEV_TX_OFFLOAD_TCP_TSO;
1426
1427         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1428         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1429
1430         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1431                 .rx_thresh = {
1432                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1433                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1434                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1435                 },
1436                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1437                 .rx_drop_en = 0,
1438         };
1439
1440         dev_info->default_txconf = (struct rte_eth_txconf) {
1441                 .tx_thresh = {
1442                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1443                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1444                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1445                 },
1446                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1447                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1448                 .txq_flags = FM10K_SIMPLE_TX_FLAG,
1449         };
1450
1451         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1452                 .nb_max = FM10K_MAX_RX_DESC,
1453                 .nb_min = FM10K_MIN_RX_DESC,
1454                 .nb_align = FM10K_MULT_RX_DESC,
1455         };
1456
1457         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1458                 .nb_max = FM10K_MAX_TX_DESC,
1459                 .nb_min = FM10K_MIN_TX_DESC,
1460                 .nb_align = FM10K_MULT_TX_DESC,
1461                 .nb_seg_max = FM10K_TX_MAX_SEG,
1462                 .nb_mtu_seg_max = FM10K_TX_MAX_MTU_SEG,
1463         };
1464
1465         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
1466                         ETH_LINK_SPEED_10G | ETH_LINK_SPEED_25G |
1467                         ETH_LINK_SPEED_40G | ETH_LINK_SPEED_100G;
1468 }
1469
1470 #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
1471 static const uint32_t *
1472 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1473 {
1474         if (dev->rx_pkt_burst == fm10k_recv_pkts ||
1475             dev->rx_pkt_burst == fm10k_recv_scattered_pkts) {
1476                 static uint32_t ptypes[] = {
1477                         /* refers to rx_desc_to_ol_flags() */
1478                         RTE_PTYPE_L2_ETHER,
1479                         RTE_PTYPE_L3_IPV4,
1480                         RTE_PTYPE_L3_IPV4_EXT,
1481                         RTE_PTYPE_L3_IPV6,
1482                         RTE_PTYPE_L3_IPV6_EXT,
1483                         RTE_PTYPE_L4_TCP,
1484                         RTE_PTYPE_L4_UDP,
1485                         RTE_PTYPE_UNKNOWN
1486                 };
1487
1488                 return ptypes;
1489         } else if (dev->rx_pkt_burst == fm10k_recv_pkts_vec ||
1490                    dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec) {
1491                 static uint32_t ptypes_vec[] = {
1492                         /* refers to fm10k_desc_to_pktype_v() */
1493                         RTE_PTYPE_L3_IPV4,
1494                         RTE_PTYPE_L3_IPV4_EXT,
1495                         RTE_PTYPE_L3_IPV6,
1496                         RTE_PTYPE_L3_IPV6_EXT,
1497                         RTE_PTYPE_L4_TCP,
1498                         RTE_PTYPE_L4_UDP,
1499                         RTE_PTYPE_TUNNEL_GENEVE,
1500                         RTE_PTYPE_TUNNEL_NVGRE,
1501                         RTE_PTYPE_TUNNEL_VXLAN,
1502                         RTE_PTYPE_TUNNEL_GRE,
1503                         RTE_PTYPE_UNKNOWN
1504                 };
1505
1506                 return ptypes_vec;
1507         }
1508
1509         return NULL;
1510 }
1511 #else
1512 static const uint32_t *
1513 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1514 {
1515         return NULL;
1516 }
1517 #endif
1518
1519 static int
1520 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1521 {
1522         s32 result;
1523         uint16_t mac_num = 0;
1524         uint32_t vid_idx, vid_bit, mac_index;
1525         struct fm10k_hw *hw;
1526         struct fm10k_macvlan_filter_info *macvlan;
1527         struct rte_eth_dev_data *data = dev->data;
1528
1529         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1530         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1531
1532         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1533                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1534                 return -EINVAL;
1535         }
1536
1537         if (vlan_id > ETH_VLAN_ID_MAX) {
1538                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1539                 return -EINVAL;
1540         }
1541
1542         vid_idx = FM10K_VFTA_IDX(vlan_id);
1543         vid_bit = FM10K_VFTA_BIT(vlan_id);
1544         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1545         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1546                 return 0;
1547         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1548         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1549                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1550                         "in the VLAN filter table");
1551                 return -EINVAL;
1552         }
1553
1554         fm10k_mbx_lock(hw);
1555         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1556         fm10k_mbx_unlock(hw);
1557         if (result != FM10K_SUCCESS) {
1558                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1559                 return -EIO;
1560         }
1561
1562         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1563                         (result == FM10K_SUCCESS); mac_index++) {
1564                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1565                         continue;
1566                 if (mac_num > macvlan->mac_num - 1) {
1567                         PMD_INIT_LOG(ERR, "MAC address number "
1568                                         "not match");
1569                         break;
1570                 }
1571                 fm10k_mbx_lock(hw);
1572                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1573                         data->mac_addrs[mac_index].addr_bytes,
1574                         vlan_id, on, 0);
1575                 fm10k_mbx_unlock(hw);
1576                 mac_num++;
1577         }
1578         if (result != FM10K_SUCCESS) {
1579                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1580                 return -EIO;
1581         }
1582
1583         if (on) {
1584                 macvlan->vlan_num++;
1585                 macvlan->vfta[vid_idx] |= vid_bit;
1586         } else {
1587                 macvlan->vlan_num--;
1588                 macvlan->vfta[vid_idx] &= ~vid_bit;
1589         }
1590         return 0;
1591 }
1592
1593 static void
1594 fm10k_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1595 {
1596         if (mask & ETH_VLAN_STRIP_MASK) {
1597                 if (!dev->data->dev_conf.rxmode.hw_vlan_strip)
1598                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1599                                         "always on in fm10k");
1600         }
1601
1602         if (mask & ETH_VLAN_EXTEND_MASK) {
1603                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1604                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1605                                         "supported in fm10k");
1606         }
1607
1608         if (mask & ETH_VLAN_FILTER_MASK) {
1609                 if (!dev->data->dev_conf.rxmode.hw_vlan_filter)
1610                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1611         }
1612 }
1613
1614 /* Add/Remove a MAC address, and update filters to main VSI */
1615 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1616                 const u8 *mac, bool add, uint32_t pool)
1617 {
1618         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1619         struct fm10k_macvlan_filter_info *macvlan;
1620         uint32_t i, j, k;
1621
1622         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1623
1624         if (pool != MAIN_VSI_POOL_NUMBER) {
1625                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1626                         "mac to pool %u", pool);
1627                 return;
1628         }
1629         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1630                 if (!macvlan->vfta[j])
1631                         continue;
1632                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1633                         if (!(macvlan->vfta[j] & (1 << k)))
1634                                 continue;
1635                         if (i + 1 > macvlan->vlan_num) {
1636                                 PMD_INIT_LOG(ERR, "vlan number not match");
1637                                 return;
1638                         }
1639                         fm10k_mbx_lock(hw);
1640                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1641                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1642                         fm10k_mbx_unlock(hw);
1643                         i++;
1644                 }
1645         }
1646 }
1647
1648 /* Add/Remove a MAC address, and update filters to VMDQ */
1649 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1650                 const u8 *mac, bool add, uint32_t pool)
1651 {
1652         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1653         struct fm10k_macvlan_filter_info *macvlan;
1654         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1655         uint32_t i;
1656
1657         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1658         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1659
1660         if (pool > macvlan->nb_queue_pools) {
1661                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1662                         " Max pool is %u",
1663                         pool, macvlan->nb_queue_pools);
1664                 return;
1665         }
1666         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1667                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1668                         continue;
1669                 fm10k_mbx_lock(hw);
1670                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1671                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1672                 fm10k_mbx_unlock(hw);
1673         }
1674 }
1675
1676 /* Add/Remove a MAC address, and update filters */
1677 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1678                 const u8 *mac, bool add, uint32_t pool)
1679 {
1680         struct fm10k_macvlan_filter_info *macvlan;
1681
1682         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1683
1684         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1685                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1686         else
1687                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1688
1689         if (add)
1690                 macvlan->mac_num++;
1691         else
1692                 macvlan->mac_num--;
1693 }
1694
1695 /* Add a MAC address, and update filters */
1696 static int
1697 fm10k_macaddr_add(struct rte_eth_dev *dev,
1698                 struct ether_addr *mac_addr,
1699                 uint32_t index,
1700                 uint32_t pool)
1701 {
1702         struct fm10k_macvlan_filter_info *macvlan;
1703
1704         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1705         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1706         macvlan->mac_vmdq_id[index] = pool;
1707         return 0;
1708 }
1709
1710 /* Remove a MAC address, and update filters */
1711 static void
1712 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1713 {
1714         struct rte_eth_dev_data *data = dev->data;
1715         struct fm10k_macvlan_filter_info *macvlan;
1716
1717         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1718         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1719                         FALSE, macvlan->mac_vmdq_id[index]);
1720         macvlan->mac_vmdq_id[index] = 0;
1721 }
1722
1723 static inline int
1724 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1725 {
1726         if ((request < min) || (request > max) || ((request % mult) != 0))
1727                 return -1;
1728         else
1729                 return 0;
1730 }
1731
1732
1733 static inline int
1734 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1735 {
1736         if ((request < min) || (request > max) || ((div % request) != 0))
1737                 return -1;
1738         else
1739                 return 0;
1740 }
1741
1742 static inline int
1743 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1744 {
1745         uint16_t rx_free_thresh;
1746
1747         if (conf->rx_free_thresh == 0)
1748                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1749         else
1750                 rx_free_thresh = conf->rx_free_thresh;
1751
1752         /* make sure the requested threshold satisfies the constraints */
1753         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1754                         FM10K_RX_FREE_THRESH_MAX(q),
1755                         FM10K_RX_FREE_THRESH_DIV(q),
1756                         rx_free_thresh)) {
1757                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1758                         "less than or equal to %u, "
1759                         "greater than or equal to %u, "
1760                         "and a divisor of %u",
1761                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1762                         FM10K_RX_FREE_THRESH_MIN(q),
1763                         FM10K_RX_FREE_THRESH_DIV(q));
1764                 return -EINVAL;
1765         }
1766
1767         q->alloc_thresh = rx_free_thresh;
1768         q->drop_en = conf->rx_drop_en;
1769         q->rx_deferred_start = conf->rx_deferred_start;
1770
1771         return 0;
1772 }
1773
1774 /*
1775  * Hardware requires specific alignment for Rx packet buffers. At
1776  * least one of the following two conditions must be satisfied.
1777  *  1. Address is 512B aligned
1778  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1779  *
1780  * As such, the driver may need to adjust the DMA address within the
1781  * buffer by up to 512B.
1782  *
1783  * return 1 if the element size is valid, otherwise return 0.
1784  */
1785 static int
1786 mempool_element_size_valid(struct rte_mempool *mp)
1787 {
1788         uint32_t min_size;
1789
1790         /* elt_size includes mbuf header and headroom */
1791         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1792                         RTE_PKTMBUF_HEADROOM;
1793
1794         /* account for up to 512B of alignment */
1795         min_size -= FM10K_RX_DATABUF_ALIGN;
1796
1797         /* sanity check for overflow */
1798         if (min_size > mp->elt_size)
1799                 return 0;
1800
1801         /* size is valid */
1802         return 1;
1803 }
1804
1805 static int
1806 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1807         uint16_t nb_desc, unsigned int socket_id,
1808         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1809 {
1810         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1811         struct fm10k_dev_info *dev_info =
1812                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1813         struct fm10k_rx_queue *q;
1814         const struct rte_memzone *mz;
1815
1816         PMD_INIT_FUNC_TRACE();
1817
1818         /* make sure the mempool element size can account for alignment. */
1819         if (!mempool_element_size_valid(mp)) {
1820                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1821                 return -EINVAL;
1822         }
1823
1824         /* make sure a valid number of descriptors have been requested */
1825         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1826                                 FM10K_MULT_RX_DESC, nb_desc)) {
1827                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1828                         "less than or equal to %"PRIu32", "
1829                         "greater than or equal to %u, "
1830                         "and a multiple of %u",
1831                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1832                         FM10K_MULT_RX_DESC);
1833                 return -EINVAL;
1834         }
1835
1836         /*
1837          * if this queue existed already, free the associated memory. The
1838          * queue cannot be reused in case we need to allocate memory on
1839          * different socket than was previously used.
1840          */
1841         if (dev->data->rx_queues[queue_id] != NULL) {
1842                 rx_queue_free(dev->data->rx_queues[queue_id]);
1843                 dev->data->rx_queues[queue_id] = NULL;
1844         }
1845
1846         /* allocate memory for the queue structure */
1847         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1848                                 socket_id);
1849         if (q == NULL) {
1850                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1851                 return -ENOMEM;
1852         }
1853
1854         /* setup queue */
1855         q->mp = mp;
1856         q->nb_desc = nb_desc;
1857         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1858         q->port_id = dev->data->port_id;
1859         q->queue_id = queue_id;
1860         q->tail_ptr = (volatile uint32_t *)
1861                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1862         if (handle_rxconf(q, conf))
1863                 return -EINVAL;
1864
1865         /* allocate memory for the software ring */
1866         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1867                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1868                         RTE_CACHE_LINE_SIZE, socket_id);
1869         if (q->sw_ring == NULL) {
1870                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1871                 rte_free(q);
1872                 return -ENOMEM;
1873         }
1874
1875         /*
1876          * allocate memory for the hardware descriptor ring. A memzone large
1877          * enough to hold the maximum ring size is requested to allow for
1878          * resizing in later calls to the queue setup function.
1879          */
1880         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1881                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1882                                       socket_id);
1883         if (mz == NULL) {
1884                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1885                 rte_free(q->sw_ring);
1886                 rte_free(q);
1887                 return -ENOMEM;
1888         }
1889         q->hw_ring = mz->addr;
1890         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1891
1892         /* Check if number of descs satisfied Vector requirement */
1893         if (!rte_is_power_of_2(nb_desc)) {
1894                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1895                                     "preconditions - canceling the feature for "
1896                                     "the whole port[%d]",
1897                              q->queue_id, q->port_id);
1898                 dev_info->rx_vec_allowed = false;
1899         } else
1900                 fm10k_rxq_vec_setup(q);
1901
1902         dev->data->rx_queues[queue_id] = q;
1903         return 0;
1904 }
1905
1906 static void
1907 fm10k_rx_queue_release(void *queue)
1908 {
1909         PMD_INIT_FUNC_TRACE();
1910
1911         rx_queue_free(queue);
1912 }
1913
1914 static inline int
1915 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1916 {
1917         uint16_t tx_free_thresh;
1918         uint16_t tx_rs_thresh;
1919
1920         /* constraint MACROs require that tx_free_thresh is configured
1921          * before tx_rs_thresh */
1922         if (conf->tx_free_thresh == 0)
1923                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1924         else
1925                 tx_free_thresh = conf->tx_free_thresh;
1926
1927         /* make sure the requested threshold satisfies the constraints */
1928         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1929                         FM10K_TX_FREE_THRESH_MAX(q),
1930                         FM10K_TX_FREE_THRESH_DIV(q),
1931                         tx_free_thresh)) {
1932                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1933                         "less than or equal to %u, "
1934                         "greater than or equal to %u, "
1935                         "and a divisor of %u",
1936                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1937                         FM10K_TX_FREE_THRESH_MIN(q),
1938                         FM10K_TX_FREE_THRESH_DIV(q));
1939                 return -EINVAL;
1940         }
1941
1942         q->free_thresh = tx_free_thresh;
1943
1944         if (conf->tx_rs_thresh == 0)
1945                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1946         else
1947                 tx_rs_thresh = conf->tx_rs_thresh;
1948
1949         q->tx_deferred_start = conf->tx_deferred_start;
1950
1951         /* make sure the requested threshold satisfies the constraints */
1952         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1953                         FM10K_TX_RS_THRESH_MAX(q),
1954                         FM10K_TX_RS_THRESH_DIV(q),
1955                         tx_rs_thresh)) {
1956                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1957                         "less than or equal to %u, "
1958                         "greater than or equal to %u, "
1959                         "and a divisor of %u",
1960                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1961                         FM10K_TX_RS_THRESH_MIN(q),
1962                         FM10K_TX_RS_THRESH_DIV(q));
1963                 return -EINVAL;
1964         }
1965
1966         q->rs_thresh = tx_rs_thresh;
1967
1968         return 0;
1969 }
1970
1971 static int
1972 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1973         uint16_t nb_desc, unsigned int socket_id,
1974         const struct rte_eth_txconf *conf)
1975 {
1976         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1977         struct fm10k_tx_queue *q;
1978         const struct rte_memzone *mz;
1979
1980         PMD_INIT_FUNC_TRACE();
1981
1982         /* make sure a valid number of descriptors have been requested */
1983         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1984                                 FM10K_MULT_TX_DESC, nb_desc)) {
1985                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1986                         "less than or equal to %"PRIu32", "
1987                         "greater than or equal to %u, "
1988                         "and a multiple of %u",
1989                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1990                         FM10K_MULT_TX_DESC);
1991                 return -EINVAL;
1992         }
1993
1994         /*
1995          * if this queue existed already, free the associated memory. The
1996          * queue cannot be reused in case we need to allocate memory on
1997          * different socket than was previously used.
1998          */
1999         if (dev->data->tx_queues[queue_id] != NULL) {
2000                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
2001
2002                 tx_queue_free(txq);
2003                 dev->data->tx_queues[queue_id] = NULL;
2004         }
2005
2006         /* allocate memory for the queue structure */
2007         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
2008                                 socket_id);
2009         if (q == NULL) {
2010                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
2011                 return -ENOMEM;
2012         }
2013
2014         /* setup queue */
2015         q->nb_desc = nb_desc;
2016         q->port_id = dev->data->port_id;
2017         q->queue_id = queue_id;
2018         q->txq_flags = conf->txq_flags;
2019         q->ops = &def_txq_ops;
2020         q->tail_ptr = (volatile uint32_t *)
2021                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
2022         if (handle_txconf(q, conf))
2023                 return -EINVAL;
2024
2025         /* allocate memory for the software ring */
2026         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
2027                                         nb_desc * sizeof(struct rte_mbuf *),
2028                                         RTE_CACHE_LINE_SIZE, socket_id);
2029         if (q->sw_ring == NULL) {
2030                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
2031                 rte_free(q);
2032                 return -ENOMEM;
2033         }
2034
2035         /*
2036          * allocate memory for the hardware descriptor ring. A memzone large
2037          * enough to hold the maximum ring size is requested to allow for
2038          * resizing in later calls to the queue setup function.
2039          */
2040         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
2041                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
2042                                       socket_id);
2043         if (mz == NULL) {
2044                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
2045                 rte_free(q->sw_ring);
2046                 rte_free(q);
2047                 return -ENOMEM;
2048         }
2049         q->hw_ring = mz->addr;
2050         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
2051
2052         /*
2053          * allocate memory for the RS bit tracker. Enough slots to hold the
2054          * descriptor index for each RS bit needing to be set are required.
2055          */
2056         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
2057                                 ((nb_desc + 1) / q->rs_thresh) *
2058                                 sizeof(uint16_t),
2059                                 RTE_CACHE_LINE_SIZE, socket_id);
2060         if (q->rs_tracker.list == NULL) {
2061                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
2062                 rte_free(q->sw_ring);
2063                 rte_free(q);
2064                 return -ENOMEM;
2065         }
2066
2067         dev->data->tx_queues[queue_id] = q;
2068         return 0;
2069 }
2070
2071 static void
2072 fm10k_tx_queue_release(void *queue)
2073 {
2074         struct fm10k_tx_queue *q = queue;
2075         PMD_INIT_FUNC_TRACE();
2076
2077         tx_queue_free(q);
2078 }
2079
2080 static int
2081 fm10k_reta_update(struct rte_eth_dev *dev,
2082                         struct rte_eth_rss_reta_entry64 *reta_conf,
2083                         uint16_t reta_size)
2084 {
2085         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2086         uint16_t i, j, idx, shift;
2087         uint8_t mask;
2088         uint32_t reta;
2089
2090         PMD_INIT_FUNC_TRACE();
2091
2092         if (reta_size > FM10K_MAX_RSS_INDICES) {
2093                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2094                         "(%d) doesn't match the number hardware can supported "
2095                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2096                 return -EINVAL;
2097         }
2098
2099         /*
2100          * Update Redirection Table RETA[n], n=0..31. The redirection table has
2101          * 128-entries in 32 registers
2102          */
2103         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2104                 idx = i / RTE_RETA_GROUP_SIZE;
2105                 shift = i % RTE_RETA_GROUP_SIZE;
2106                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2107                                 BIT_MASK_PER_UINT32);
2108                 if (mask == 0)
2109                         continue;
2110
2111                 reta = 0;
2112                 if (mask != BIT_MASK_PER_UINT32)
2113                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2114
2115                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2116                         if (mask & (0x1 << j)) {
2117                                 if (mask != 0xF)
2118                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
2119                                 reta |= reta_conf[idx].reta[shift + j] <<
2120                                                 (CHAR_BIT * j);
2121                         }
2122                 }
2123                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2124         }
2125
2126         return 0;
2127 }
2128
2129 static int
2130 fm10k_reta_query(struct rte_eth_dev *dev,
2131                         struct rte_eth_rss_reta_entry64 *reta_conf,
2132                         uint16_t reta_size)
2133 {
2134         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2135         uint16_t i, j, idx, shift;
2136         uint8_t mask;
2137         uint32_t reta;
2138
2139         PMD_INIT_FUNC_TRACE();
2140
2141         if (reta_size < FM10K_MAX_RSS_INDICES) {
2142                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2143                         "(%d) doesn't match the number hardware can supported "
2144                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2145                 return -EINVAL;
2146         }
2147
2148         /*
2149          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2150          * 128-entries in 32 registers
2151          */
2152         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2153                 idx = i / RTE_RETA_GROUP_SIZE;
2154                 shift = i % RTE_RETA_GROUP_SIZE;
2155                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2156                                 BIT_MASK_PER_UINT32);
2157                 if (mask == 0)
2158                         continue;
2159
2160                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2161                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2162                         if (mask & (0x1 << j))
2163                                 reta_conf[idx].reta[shift + j] = ((reta >>
2164                                         CHAR_BIT * j) & UINT8_MAX);
2165                 }
2166         }
2167
2168         return 0;
2169 }
2170
2171 static int
2172 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2173         struct rte_eth_rss_conf *rss_conf)
2174 {
2175         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2176         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2177         uint32_t mrqc;
2178         uint64_t hf = rss_conf->rss_hf;
2179         int i;
2180
2181         PMD_INIT_FUNC_TRACE();
2182
2183         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2184                                 FM10K_RSSRK_ENTRIES_PER_REG))
2185                 return -EINVAL;
2186
2187         if (hf == 0)
2188                 return -EINVAL;
2189
2190         mrqc = 0;
2191         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2192         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2193         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2194         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2195         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2196         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2197         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2198         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2199         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2200
2201         /* If the mapping doesn't fit any supported, return */
2202         if (mrqc == 0)
2203                 return -EINVAL;
2204
2205         if (key != NULL)
2206                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2207                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2208
2209         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2210
2211         return 0;
2212 }
2213
2214 static int
2215 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2216         struct rte_eth_rss_conf *rss_conf)
2217 {
2218         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2219         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2220         uint32_t mrqc;
2221         uint64_t hf;
2222         int i;
2223
2224         PMD_INIT_FUNC_TRACE();
2225
2226         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2227                                 FM10K_RSSRK_ENTRIES_PER_REG))
2228                 return -EINVAL;
2229
2230         if (key != NULL)
2231                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2232                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2233
2234         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2235         hf = 0;
2236         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2237         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2238         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2239         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2240         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2241         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2242         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2243         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2244         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2245
2246         rss_conf->rss_hf = hf;
2247
2248         return 0;
2249 }
2250
2251 static void
2252 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2253 {
2254         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2255         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2256
2257         /* Bind all local non-queue interrupt to vector 0 */
2258         int_map |= FM10K_MISC_VEC_ID;
2259
2260         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2261         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2262         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2263         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2264         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2265         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2266
2267         /* Enable misc causes */
2268         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2269                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2270                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2271                                 FM10K_EIMR_ENABLE(MAILBOX) |
2272                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2273                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2274                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2275                                 FM10K_EIMR_ENABLE(VFLR));
2276
2277         /* Enable ITR 0 */
2278         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2279                                         FM10K_ITR_MASK_CLEAR);
2280         FM10K_WRITE_FLUSH(hw);
2281 }
2282
2283 static void
2284 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2285 {
2286         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2287         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2288
2289         int_map |= FM10K_MISC_VEC_ID;
2290
2291         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2292         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2293         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2294         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2295         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2296         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2297
2298         /* Disable misc causes */
2299         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2300                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2301                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2302                                 FM10K_EIMR_DISABLE(MAILBOX) |
2303                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2304                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2305                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2306                                 FM10K_EIMR_DISABLE(VFLR));
2307
2308         /* Disable ITR 0 */
2309         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2310         FM10K_WRITE_FLUSH(hw);
2311 }
2312
2313 static void
2314 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2315 {
2316         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2317         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2318
2319         /* Bind all local non-queue interrupt to vector 0 */
2320         int_map |= FM10K_MISC_VEC_ID;
2321
2322         /* Only INT 0 available, other 15 are reserved. */
2323         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2324
2325         /* Enable ITR 0 */
2326         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2327                                         FM10K_ITR_MASK_CLEAR);
2328         FM10K_WRITE_FLUSH(hw);
2329 }
2330
2331 static void
2332 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2333 {
2334         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2335         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2336
2337         int_map |= FM10K_MISC_VEC_ID;
2338
2339         /* Only INT 0 available, other 15 are reserved. */
2340         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2341
2342         /* Disable ITR 0 */
2343         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2344         FM10K_WRITE_FLUSH(hw);
2345 }
2346
2347 static int
2348 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2349 {
2350         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2351         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2352
2353         /* Enable ITR */
2354         if (hw->mac.type == fm10k_mac_pf)
2355                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2356                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2357         else
2358                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2359                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2360         rte_intr_enable(&pdev->intr_handle);
2361         return 0;
2362 }
2363
2364 static int
2365 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2366 {
2367         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2368         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2369
2370         /* Disable ITR */
2371         if (hw->mac.type == fm10k_mac_pf)
2372                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2373                         FM10K_ITR_MASK_SET);
2374         else
2375                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2376                         FM10K_ITR_MASK_SET);
2377         return 0;
2378 }
2379
2380 static int
2381 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2382 {
2383         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2384         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2385         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2386         uint32_t intr_vector, vec;
2387         uint16_t queue_id;
2388         int result = 0;
2389
2390         /* fm10k needs one separate interrupt for mailbox,
2391          * so only drivers which support multiple interrupt vectors
2392          * e.g. vfio-pci can work for fm10k interrupt mode
2393          */
2394         if (!rte_intr_cap_multiple(intr_handle) ||
2395                         dev->data->dev_conf.intr_conf.rxq == 0)
2396                 return result;
2397
2398         intr_vector = dev->data->nb_rx_queues;
2399
2400         /* disable interrupt first */
2401         rte_intr_disable(intr_handle);
2402         if (hw->mac.type == fm10k_mac_pf)
2403                 fm10k_dev_disable_intr_pf(dev);
2404         else
2405                 fm10k_dev_disable_intr_vf(dev);
2406
2407         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2408                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2409                 result = -EIO;
2410         }
2411
2412         if (rte_intr_dp_is_en(intr_handle) && !result) {
2413                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2414                         dev->data->nb_rx_queues * sizeof(int), 0);
2415                 if (intr_handle->intr_vec) {
2416                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2417                                         queue_id < dev->data->nb_rx_queues;
2418                                         queue_id++) {
2419                                 intr_handle->intr_vec[queue_id] = vec;
2420                                 if (vec < intr_handle->nb_efd - 1
2421                                                 + FM10K_RX_VEC_START)
2422                                         vec++;
2423                         }
2424                 } else {
2425                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2426                                 " intr_vec", dev->data->nb_rx_queues);
2427                         rte_intr_efd_disable(intr_handle);
2428                         result = -ENOMEM;
2429                 }
2430         }
2431
2432         if (hw->mac.type == fm10k_mac_pf)
2433                 fm10k_dev_enable_intr_pf(dev);
2434         else
2435                 fm10k_dev_enable_intr_vf(dev);
2436         rte_intr_enable(intr_handle);
2437         hw->mac.ops.update_int_moderator(hw);
2438         return result;
2439 }
2440
2441 static int
2442 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2443 {
2444         struct fm10k_fault fault;
2445         int err;
2446         const char *estr = "Unknown error";
2447
2448         /* Process PCA fault */
2449         if (eicr & FM10K_EICR_PCA_FAULT) {
2450                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2451                 if (err)
2452                         goto error;
2453                 switch (fault.type) {
2454                 case PCA_NO_FAULT:
2455                         estr = "PCA_NO_FAULT"; break;
2456                 case PCA_UNMAPPED_ADDR:
2457                         estr = "PCA_UNMAPPED_ADDR"; break;
2458                 case PCA_BAD_QACCESS_PF:
2459                         estr = "PCA_BAD_QACCESS_PF"; break;
2460                 case PCA_BAD_QACCESS_VF:
2461                         estr = "PCA_BAD_QACCESS_VF"; break;
2462                 case PCA_MALICIOUS_REQ:
2463                         estr = "PCA_MALICIOUS_REQ"; break;
2464                 case PCA_POISONED_TLP:
2465                         estr = "PCA_POISONED_TLP"; break;
2466                 case PCA_TLP_ABORT:
2467                         estr = "PCA_TLP_ABORT"; break;
2468                 default:
2469                         goto error;
2470                 }
2471                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2472                         estr, fault.func ? "VF" : "PF", fault.func,
2473                         fault.address, fault.specinfo);
2474         }
2475
2476         /* Process THI fault */
2477         if (eicr & FM10K_EICR_THI_FAULT) {
2478                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2479                 if (err)
2480                         goto error;
2481                 switch (fault.type) {
2482                 case THI_NO_FAULT:
2483                         estr = "THI_NO_FAULT"; break;
2484                 case THI_MAL_DIS_Q_FAULT:
2485                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2486                 default:
2487                         goto error;
2488                 }
2489                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2490                         estr, fault.func ? "VF" : "PF", fault.func,
2491                         fault.address, fault.specinfo);
2492         }
2493
2494         /* Process FUM fault */
2495         if (eicr & FM10K_EICR_FUM_FAULT) {
2496                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2497                 if (err)
2498                         goto error;
2499                 switch (fault.type) {
2500                 case FUM_NO_FAULT:
2501                         estr = "FUM_NO_FAULT"; break;
2502                 case FUM_UNMAPPED_ADDR:
2503                         estr = "FUM_UNMAPPED_ADDR"; break;
2504                 case FUM_POISONED_TLP:
2505                         estr = "FUM_POISONED_TLP"; break;
2506                 case FUM_BAD_VF_QACCESS:
2507                         estr = "FUM_BAD_VF_QACCESS"; break;
2508                 case FUM_ADD_DECODE_ERR:
2509                         estr = "FUM_ADD_DECODE_ERR"; break;
2510                 case FUM_RO_ERROR:
2511                         estr = "FUM_RO_ERROR"; break;
2512                 case FUM_QPRC_CRC_ERROR:
2513                         estr = "FUM_QPRC_CRC_ERROR"; break;
2514                 case FUM_CSR_TIMEOUT:
2515                         estr = "FUM_CSR_TIMEOUT"; break;
2516                 case FUM_INVALID_TYPE:
2517                         estr = "FUM_INVALID_TYPE"; break;
2518                 case FUM_INVALID_LENGTH:
2519                         estr = "FUM_INVALID_LENGTH"; break;
2520                 case FUM_INVALID_BE:
2521                         estr = "FUM_INVALID_BE"; break;
2522                 case FUM_INVALID_ALIGN:
2523                         estr = "FUM_INVALID_ALIGN"; break;
2524                 default:
2525                         goto error;
2526                 }
2527                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2528                         estr, fault.func ? "VF" : "PF", fault.func,
2529                         fault.address, fault.specinfo);
2530         }
2531
2532         return 0;
2533 error:
2534         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2535         return err;
2536 }
2537
2538 /**
2539  * PF interrupt handler triggered by NIC for handling specific interrupt.
2540  *
2541  * @param handle
2542  *  Pointer to interrupt handle.
2543  * @param param
2544  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2545  *
2546  * @return
2547  *  void
2548  */
2549 static void
2550 fm10k_dev_interrupt_handler_pf(void *param)
2551 {
2552         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2553         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2554         uint32_t cause, status;
2555
2556         if (hw->mac.type != fm10k_mac_pf)
2557                 return;
2558
2559         cause = FM10K_READ_REG(hw, FM10K_EICR);
2560
2561         /* Handle PCI fault cases */
2562         if (cause & FM10K_EICR_FAULT_MASK) {
2563                 PMD_INIT_LOG(ERR, "INT: find fault!");
2564                 fm10k_dev_handle_fault(hw, cause);
2565         }
2566
2567         /* Handle switch up/down */
2568         if (cause & FM10K_EICR_SWITCHNOTREADY)
2569                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2570
2571         if (cause & FM10K_EICR_SWITCHREADY)
2572                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2573
2574         /* Handle mailbox message */
2575         fm10k_mbx_lock(hw);
2576         hw->mbx.ops.process(hw, &hw->mbx);
2577         fm10k_mbx_unlock(hw);
2578
2579         /* Handle SRAM error */
2580         if (cause & FM10K_EICR_SRAMERROR) {
2581                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2582
2583                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2584                 /* Write to clear pending bits */
2585                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2586
2587                 /* Todo: print out error message after shared code  updates */
2588         }
2589
2590         /* Clear these 3 events if having any */
2591         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2592                  FM10K_EICR_SWITCHREADY;
2593         if (cause)
2594                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2595
2596         /* Re-enable interrupt from device side */
2597         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2598                                         FM10K_ITR_MASK_CLEAR);
2599         /* Re-enable interrupt from host side */
2600         rte_intr_enable(dev->intr_handle);
2601 }
2602
2603 /**
2604  * VF interrupt handler triggered by NIC for handling specific interrupt.
2605  *
2606  * @param handle
2607  *  Pointer to interrupt handle.
2608  * @param param
2609  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2610  *
2611  * @return
2612  *  void
2613  */
2614 static void
2615 fm10k_dev_interrupt_handler_vf(void *param)
2616 {
2617         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2618         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2619
2620         if (hw->mac.type != fm10k_mac_vf)
2621                 return;
2622
2623         /* Handle mailbox message if lock is acquired */
2624         fm10k_mbx_lock(hw);
2625         hw->mbx.ops.process(hw, &hw->mbx);
2626         fm10k_mbx_unlock(hw);
2627
2628         /* Re-enable interrupt from device side */
2629         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2630                                         FM10K_ITR_MASK_CLEAR);
2631         /* Re-enable interrupt from host side */
2632         rte_intr_enable(dev->intr_handle);
2633 }
2634
2635 /* Mailbox message handler in VF */
2636 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2637         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2638         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2639         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2640         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2641 };
2642
2643 static int
2644 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2645 {
2646         int err = 0;
2647
2648         /* Initialize mailbox lock */
2649         fm10k_mbx_initlock(hw);
2650
2651         /* Replace default message handler with new ones */
2652         if (hw->mac.type == fm10k_mac_vf)
2653                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2654
2655         if (err) {
2656                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2657                                 err);
2658                 return err;
2659         }
2660         /* Connect to SM for PF device or PF for VF device */
2661         return hw->mbx.ops.connect(hw, &hw->mbx);
2662 }
2663
2664 static void
2665 fm10k_close_mbx_service(struct fm10k_hw *hw)
2666 {
2667         /* Disconnect from SM for PF device or PF for VF device */
2668         hw->mbx.ops.disconnect(hw, &hw->mbx);
2669 }
2670
2671 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2672         .dev_configure          = fm10k_dev_configure,
2673         .dev_start              = fm10k_dev_start,
2674         .dev_stop               = fm10k_dev_stop,
2675         .dev_close              = fm10k_dev_close,
2676         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2677         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2678         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2679         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2680         .stats_get              = fm10k_stats_get,
2681         .xstats_get             = fm10k_xstats_get,
2682         .xstats_get_names       = fm10k_xstats_get_names,
2683         .stats_reset            = fm10k_stats_reset,
2684         .xstats_reset           = fm10k_stats_reset,
2685         .link_update            = fm10k_link_update,
2686         .dev_infos_get          = fm10k_dev_infos_get,
2687         .dev_supported_ptypes_get = fm10k_dev_supported_ptypes_get,
2688         .vlan_filter_set        = fm10k_vlan_filter_set,
2689         .vlan_offload_set       = fm10k_vlan_offload_set,
2690         .mac_addr_add           = fm10k_macaddr_add,
2691         .mac_addr_remove        = fm10k_macaddr_remove,
2692         .rx_queue_start         = fm10k_dev_rx_queue_start,
2693         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2694         .tx_queue_start         = fm10k_dev_tx_queue_start,
2695         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2696         .rx_queue_setup         = fm10k_rx_queue_setup,
2697         .rx_queue_release       = fm10k_rx_queue_release,
2698         .tx_queue_setup         = fm10k_tx_queue_setup,
2699         .tx_queue_release       = fm10k_tx_queue_release,
2700         .rx_descriptor_done     = fm10k_dev_rx_descriptor_done,
2701         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2702         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2703         .reta_update            = fm10k_reta_update,
2704         .reta_query             = fm10k_reta_query,
2705         .rss_hash_update        = fm10k_rss_hash_update,
2706         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2707 };
2708
2709 static int ftag_check_handler(__rte_unused const char *key,
2710                 const char *value, __rte_unused void *opaque)
2711 {
2712         if (strcmp(value, "1"))
2713                 return -1;
2714
2715         return 0;
2716 }
2717
2718 static int
2719 fm10k_check_ftag(struct rte_devargs *devargs)
2720 {
2721         struct rte_kvargs *kvlist;
2722         const char *ftag_key = "enable_ftag";
2723
2724         if (devargs == NULL)
2725                 return 0;
2726
2727         kvlist = rte_kvargs_parse(devargs->args, NULL);
2728         if (kvlist == NULL)
2729                 return 0;
2730
2731         if (!rte_kvargs_count(kvlist, ftag_key)) {
2732                 rte_kvargs_free(kvlist);
2733                 return 0;
2734         }
2735         /* FTAG is enabled when there's key-value pair: enable_ftag=1 */
2736         if (rte_kvargs_process(kvlist, ftag_key,
2737                                 ftag_check_handler, NULL) < 0) {
2738                 rte_kvargs_free(kvlist);
2739                 return 0;
2740         }
2741         rte_kvargs_free(kvlist);
2742
2743         return 1;
2744 }
2745
2746 static uint16_t
2747 fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
2748                     uint16_t nb_pkts)
2749 {
2750         uint16_t nb_tx = 0;
2751         struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue;
2752
2753         while (nb_pkts) {
2754                 uint16_t ret, num;
2755
2756                 num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh);
2757                 ret = fm10k_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
2758                                                  num);
2759                 nb_tx += ret;
2760                 nb_pkts -= ret;
2761                 if (ret < num)
2762                         break;
2763         }
2764
2765         return nb_tx;
2766 }
2767
2768 static void __attribute__((cold))
2769 fm10k_set_tx_function(struct rte_eth_dev *dev)
2770 {
2771         struct fm10k_tx_queue *txq;
2772         int i;
2773         int use_sse = 1;
2774         uint16_t tx_ftag_en = 0;
2775
2776         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2777                 /* primary process has set the ftag flag and txq_flags */
2778                 txq = dev->data->tx_queues[0];
2779                 if (fm10k_tx_vec_condition_check(txq)) {
2780                         dev->tx_pkt_burst = fm10k_xmit_pkts;
2781                         dev->tx_pkt_prepare = fm10k_prep_pkts;
2782                         PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2783                 } else {
2784                         PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2785                         dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2786                         dev->tx_pkt_prepare = NULL;
2787                 }
2788                 return;
2789         }
2790
2791         if (fm10k_check_ftag(dev->device->devargs))
2792                 tx_ftag_en = 1;
2793
2794         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2795                 txq = dev->data->tx_queues[i];
2796                 txq->tx_ftag_en = tx_ftag_en;
2797                 /* Check if Vector Tx is satisfied */
2798                 if (fm10k_tx_vec_condition_check(txq))
2799                         use_sse = 0;
2800         }
2801
2802         if (use_sse) {
2803                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2804                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2805                         txq = dev->data->tx_queues[i];
2806                         fm10k_txq_vec_setup(txq);
2807                 }
2808                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2809                 dev->tx_pkt_prepare = NULL;
2810         } else {
2811                 dev->tx_pkt_burst = fm10k_xmit_pkts;
2812                 dev->tx_pkt_prepare = fm10k_prep_pkts;
2813                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2814         }
2815 }
2816
2817 static void __attribute__((cold))
2818 fm10k_set_rx_function(struct rte_eth_dev *dev)
2819 {
2820         struct fm10k_dev_info *dev_info =
2821                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2822         uint16_t i, rx_using_sse;
2823         uint16_t rx_ftag_en = 0;
2824
2825         if (fm10k_check_ftag(dev->device->devargs))
2826                 rx_ftag_en = 1;
2827
2828         /* In order to allow Vector Rx there are a few configuration
2829          * conditions to be met.
2830          */
2831         if (!fm10k_rx_vec_condition_check(dev) &&
2832                         dev_info->rx_vec_allowed && !rx_ftag_en) {
2833                 if (dev->data->scattered_rx)
2834                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
2835                 else
2836                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
2837         } else if (dev->data->scattered_rx)
2838                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
2839         else
2840                 dev->rx_pkt_burst = fm10k_recv_pkts;
2841
2842         rx_using_sse =
2843                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
2844                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
2845
2846         if (rx_using_sse)
2847                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
2848         else
2849                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
2850
2851         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2852                 return;
2853
2854         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2855                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
2856
2857                 rxq->rx_using_sse = rx_using_sse;
2858                 rxq->rx_ftag_en = rx_ftag_en;
2859         }
2860 }
2861
2862 static void
2863 fm10k_params_init(struct rte_eth_dev *dev)
2864 {
2865         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2866         struct fm10k_dev_info *info =
2867                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2868
2869         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
2870          * there is no way to get link status without reading BAR4.  Until this
2871          * works, assume we have maximum bandwidth.
2872          * @todo - fix bus info
2873          */
2874         hw->bus_caps.speed = fm10k_bus_speed_8000;
2875         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
2876         hw->bus_caps.payload = fm10k_bus_payload_512;
2877         hw->bus.speed = fm10k_bus_speed_8000;
2878         hw->bus.width = fm10k_bus_width_pcie_x8;
2879         hw->bus.payload = fm10k_bus_payload_256;
2880
2881         info->rx_vec_allowed = true;
2882 }
2883
2884 static int
2885 eth_fm10k_dev_init(struct rte_eth_dev *dev)
2886 {
2887         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2888         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2889         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2890         int diag, i;
2891         struct fm10k_macvlan_filter_info *macvlan;
2892
2893         PMD_INIT_FUNC_TRACE();
2894
2895         dev->dev_ops = &fm10k_eth_dev_ops;
2896         dev->rx_pkt_burst = &fm10k_recv_pkts;
2897         dev->tx_pkt_burst = &fm10k_xmit_pkts;
2898         dev->tx_pkt_prepare = &fm10k_prep_pkts;
2899
2900         /*
2901          * Primary process does the whole initialization, for secondary
2902          * processes, we just select the same Rx and Tx function as primary.
2903          */
2904         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2905                 fm10k_set_rx_function(dev);
2906                 fm10k_set_tx_function(dev);
2907                 return 0;
2908         }
2909
2910         rte_eth_copy_pci_info(dev, pdev);
2911         dev->data->dev_flags |= RTE_ETH_DEV_DETACHABLE;
2912
2913         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
2914         memset(macvlan, 0, sizeof(*macvlan));
2915         /* Vendor and Device ID need to be set before init of shared code */
2916         memset(hw, 0, sizeof(*hw));
2917         hw->device_id = pdev->id.device_id;
2918         hw->vendor_id = pdev->id.vendor_id;
2919         hw->subsystem_device_id = pdev->id.subsystem_device_id;
2920         hw->subsystem_vendor_id = pdev->id.subsystem_vendor_id;
2921         hw->revision_id = 0;
2922         hw->hw_addr = (void *)pdev->mem_resource[0].addr;
2923         if (hw->hw_addr == NULL) {
2924                 PMD_INIT_LOG(ERR, "Bad mem resource."
2925                         " Try to blacklist unused devices.");
2926                 return -EIO;
2927         }
2928
2929         /* Store fm10k_adapter pointer */
2930         hw->back = dev->data->dev_private;
2931
2932         /* Initialize the shared code */
2933         diag = fm10k_init_shared_code(hw);
2934         if (diag != FM10K_SUCCESS) {
2935                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
2936                 return -EIO;
2937         }
2938
2939         /* Initialize parameters */
2940         fm10k_params_init(dev);
2941
2942         /* Initialize the hw */
2943         diag = fm10k_init_hw(hw);
2944         if (diag != FM10K_SUCCESS) {
2945                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
2946                 return -EIO;
2947         }
2948
2949         /* Initialize MAC address(es) */
2950         dev->data->mac_addrs = rte_zmalloc("fm10k",
2951                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
2952         if (dev->data->mac_addrs == NULL) {
2953                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
2954                 return -ENOMEM;
2955         }
2956
2957         diag = fm10k_read_mac_addr(hw);
2958
2959         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2960                         &dev->data->mac_addrs[0]);
2961
2962         if (diag != FM10K_SUCCESS ||
2963                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
2964
2965                 /* Generate a random addr */
2966                 eth_random_addr(hw->mac.addr);
2967                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
2968                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2969                 &dev->data->mac_addrs[0]);
2970         }
2971
2972         /* Reset the hw statistics */
2973         fm10k_stats_reset(dev);
2974
2975         /* Reset the hw */
2976         diag = fm10k_reset_hw(hw);
2977         if (diag != FM10K_SUCCESS) {
2978                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
2979                 return -EIO;
2980         }
2981
2982         /* Setup mailbox service */
2983         diag = fm10k_setup_mbx_service(hw);
2984         if (diag != FM10K_SUCCESS) {
2985                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
2986                 return -EIO;
2987         }
2988
2989         /*PF/VF has different interrupt handling mechanism */
2990         if (hw->mac.type == fm10k_mac_pf) {
2991                 /* register callback func to eal lib */
2992                 rte_intr_callback_register(intr_handle,
2993                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2994
2995                 /* enable MISC interrupt */
2996                 fm10k_dev_enable_intr_pf(dev);
2997         } else { /* VF */
2998                 rte_intr_callback_register(intr_handle,
2999                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3000
3001                 fm10k_dev_enable_intr_vf(dev);
3002         }
3003
3004         /* Enable intr after callback registered */
3005         rte_intr_enable(intr_handle);
3006
3007         hw->mac.ops.update_int_moderator(hw);
3008
3009         /* Make sure Switch Manager is ready before going forward. */
3010         if (hw->mac.type == fm10k_mac_pf) {
3011                 int switch_ready = 0;
3012
3013                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3014                         fm10k_mbx_lock(hw);
3015                         hw->mac.ops.get_host_state(hw, &switch_ready);
3016                         fm10k_mbx_unlock(hw);
3017                         if (switch_ready)
3018                                 break;
3019                         /* Delay some time to acquire async LPORT_MAP info. */
3020                         rte_delay_us(WAIT_SWITCH_MSG_US);
3021                 }
3022
3023                 if (switch_ready == 0) {
3024                         PMD_INIT_LOG(ERR, "switch is not ready");
3025                         return -1;
3026                 }
3027         }
3028
3029         /*
3030          * Below function will trigger operations on mailbox, acquire lock to
3031          * avoid race condition from interrupt handler. Operations on mailbox
3032          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
3033          * will handle and generate an interrupt to our side. Then,  FIFO in
3034          * mailbox will be touched.
3035          */
3036         fm10k_mbx_lock(hw);
3037         /* Enable port first */
3038         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
3039                                         MAX_LPORT_NUM, 1);
3040
3041         /* Set unicast mode by default. App can change to other mode in other
3042          * API func.
3043          */
3044         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
3045                                         FM10K_XCAST_MODE_NONE);
3046
3047         fm10k_mbx_unlock(hw);
3048
3049         /* Make sure default VID is ready before going forward. */
3050         if (hw->mac.type == fm10k_mac_pf) {
3051                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3052                         if (hw->mac.default_vid)
3053                                 break;
3054                         /* Delay some time to acquire async port VLAN info. */
3055                         rte_delay_us(WAIT_SWITCH_MSG_US);
3056                 }
3057
3058                 if (!hw->mac.default_vid) {
3059                         PMD_INIT_LOG(ERR, "default VID is not ready");
3060                         return -1;
3061                 }
3062         }
3063
3064         /* Add default mac address */
3065         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
3066                 MAIN_VSI_POOL_NUMBER);
3067
3068         return 0;
3069 }
3070
3071 static int
3072 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
3073 {
3074         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3075         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3076         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3077         PMD_INIT_FUNC_TRACE();
3078
3079         /* only uninitialize in the primary process */
3080         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3081                 return 0;
3082
3083         /* safe to close dev here */
3084         fm10k_dev_close(dev);
3085
3086         dev->dev_ops = NULL;
3087         dev->rx_pkt_burst = NULL;
3088         dev->tx_pkt_burst = NULL;
3089
3090         /* disable uio/vfio intr */
3091         rte_intr_disable(intr_handle);
3092
3093         /*PF/VF has different interrupt handling mechanism */
3094         if (hw->mac.type == fm10k_mac_pf) {
3095                 /* disable interrupt */
3096                 fm10k_dev_disable_intr_pf(dev);
3097
3098                 /* unregister callback func to eal lib */
3099                 rte_intr_callback_unregister(intr_handle,
3100                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3101         } else {
3102                 /* disable interrupt */
3103                 fm10k_dev_disable_intr_vf(dev);
3104
3105                 rte_intr_callback_unregister(intr_handle,
3106                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3107         }
3108
3109         /* free mac memory */
3110         if (dev->data->mac_addrs) {
3111                 rte_free(dev->data->mac_addrs);
3112                 dev->data->mac_addrs = NULL;
3113         }
3114
3115         memset(hw, 0, sizeof(*hw));
3116
3117         return 0;
3118 }
3119
3120 static int eth_fm10k_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
3121         struct rte_pci_device *pci_dev)
3122 {
3123         return rte_eth_dev_pci_generic_probe(pci_dev,
3124                 sizeof(struct fm10k_adapter), eth_fm10k_dev_init);
3125 }
3126
3127 static int eth_fm10k_pci_remove(struct rte_pci_device *pci_dev)
3128 {
3129         return rte_eth_dev_pci_generic_remove(pci_dev, eth_fm10k_dev_uninit);
3130 }
3131
3132 /*
3133  * The set of PCI devices this driver supports. This driver will enable both PF
3134  * and SRIOV-VF devices.
3135  */
3136 static const struct rte_pci_id pci_id_fm10k_map[] = {
3137         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_PF) },
3138         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_SDI_FM10420_QDA2) },
3139         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_VF) },
3140         { .vendor_id = 0, /* sentinel */ },
3141 };
3142
3143 static struct rte_pci_driver rte_pmd_fm10k = {
3144         .id_table = pci_id_fm10k_map,
3145         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
3146         .probe = eth_fm10k_pci_probe,
3147         .remove = eth_fm10k_pci_remove,
3148 };
3149
3150 RTE_PMD_REGISTER_PCI(net_fm10k, rte_pmd_fm10k);
3151 RTE_PMD_REGISTER_PCI_TABLE(net_fm10k, pci_id_fm10k_map);
3152 RTE_PMD_REGISTER_KMOD_DEP(net_fm10k, "* igb_uio | uio_pci_generic | vfio-pci");