drivers/net/fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2016 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_malloc.h>
  36 #include <rte_memzone.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_dev.h>
  39 #include <rte_spinlock.h>
  40 #include <rte_kvargs.h>
  41
  42 #include "fm10k.h"
  43 #include "base/fm10k_api.h"
  44
  45 /* Default delay to acquire mailbox lock */
  46 #define FM10K_MBXLOCK_DELAY_US 20
  47 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  48
  49 #define MAIN_VSI_POOL_NUMBER 0
  50
  51 /* Max try times to acquire switch status */
  52 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  53 /* Wait interval to get switch status */
  54 #define WAIT_SWITCH_MSG_US    100000
  55 /* Number of chars per uint32 type */
  56 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  57 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  58
  59 /* default 1:1 map from queue ID to interrupt vector ID */
  60 #define Q2V(dev, queue_id) (dev->pci_dev->intr_handle.intr_vec[queue_id])
  61
  62 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  63 #define MAX_LPORT_NUM    128
  64 #define GLORT_FD_Q_BASE  0x40
  65 #define GLORT_PF_MASK    0xFFC0
  66 #define GLORT_FD_MASK    GLORT_PF_MASK
  67 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  68
  69 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  70 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  71 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  72 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  73 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  74 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  75 static int
  76 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  77 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  78         const u8 *mac, bool add, uint32_t pool);
  79 static void fm10k_tx_queue_release(void *queue);
  80 static void fm10k_rx_queue_release(void *queue);
  81 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  82 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  83 static int fm10k_check_ftag(struct rte_devargs *devargs);
  84
  85 struct fm10k_xstats_name_off {
  86         char name[RTE_ETH_XSTATS_NAME_SIZE];
  87         unsigned offset;
  88 };
  89
  90 struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  91         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  92         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  93         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  94         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  95         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  96         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
  97         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
  98         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
  99                 nodesc_drop)},
 100 };
 101
 102 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
 103                 sizeof(fm10k_hw_stats_strings[0]))
 104
 105 struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
 106         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
 107         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
 108         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
 109 };
 110
 111 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
 112                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
 113
 114 struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
 115         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 116         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 117 };
 118
 119 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 120                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 121
 122 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 123                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 124 static int
 125 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 126
 127 static void
 128 fm10k_mbx_initlock(struct fm10k_hw *hw)
 129 {
 130         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 131 }
 132
 133 static void
 134 fm10k_mbx_lock(struct fm10k_hw *hw)
 135 {
 136         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 137                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 138 }
 139
 140 static void
 141 fm10k_mbx_unlock(struct fm10k_hw *hw)
 142 {
 143         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 144 }
 145
 146 /* Stubs needed for linkage when vPMD is disabled */
 147 int __attribute__((weak))
 148 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 149 {
 150         return -1;
 151 }
 152
 153 uint16_t __attribute__((weak))
 154 fm10k_recv_pkts_vec(
 155         __rte_unused void *rx_queue,
 156         __rte_unused struct rte_mbuf **rx_pkts,
 157         __rte_unused uint16_t nb_pkts)
 158 {
 159         return 0;
 160 }
 161
 162 uint16_t __attribute__((weak))
 163 fm10k_recv_scattered_pkts_vec(
 164                 __rte_unused void *rx_queue,
 165                 __rte_unused struct rte_mbuf **rx_pkts,
 166                 __rte_unused uint16_t nb_pkts)
 167 {
 168         return 0;
 169 }
 170
 171 int __attribute__((weak))
 172 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 173
 174 {
 175         return -1;
 176 }
 177
 178 void __attribute__((weak))
 179 fm10k_rx_queue_release_mbufs_vec(
 180                 __rte_unused struct fm10k_rx_queue *rxq)
 181 {
 182         return;
 183 }
 184
 185 void __attribute__((weak))
 186 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 187 {
 188         return;
 189 }
 190
 191 int __attribute__((weak))
 192 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 193 {
 194         return -1;
 195 }
 196
 197 uint16_t __attribute__((weak))
 198 fm10k_xmit_pkts_vec(__rte_unused void *tx_queue,
 199                 __rte_unused struct rte_mbuf **tx_pkts,
 200                 __rte_unused uint16_t nb_pkts)
 201 {
 202         return 0;
 203 }
 204
 205 /*
 206  * reset queue to initial state, allocate software buffers used when starting
 207  * device.
 208  * return 0 on success
 209  * return -ENOMEM if buffers cannot be allocated
 210  * return -EINVAL if buffers do not satisfy alignment condition
 211  */
 212 static inline int
 213 rx_queue_reset(struct fm10k_rx_queue *q)
 214 {
 215         static const union fm10k_rx_desc zero = {{0} };
 216         uint64_t dma_addr;
 217         int i, diag;
 218         PMD_INIT_FUNC_TRACE();
 219
 220         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 221         if (diag != 0)
 222                 return -ENOMEM;
 223
 224         for (i = 0; i < q->nb_desc; ++i) {
 225                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 226                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 227                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 228                                                 q->nb_desc);
 229                         return -EINVAL;
 230                 }
 231                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 232                 q->hw_ring[i].q.pkt_addr = dma_addr;
 233                 q->hw_ring[i].q.hdr_addr = dma_addr;
 234         }
 235
 236         /* initialize extra software ring entries. Space for these extra
 237          * entries is always allocated.
 238          */
 239         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 240         for (i = 0; i < q->nb_fake_desc; ++i) {
 241                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 242                 q->hw_ring[q->nb_desc + i] = zero;
 243         }
 244
 245         q->next_dd = 0;
 246         q->next_alloc = 0;
 247         q->next_trigger = q->alloc_thresh - 1;
 248         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 249         q->rxrearm_start = 0;
 250         q->rxrearm_nb = 0;
 251
 252         return 0;
 253 }
 254
 255 /*
 256  * clean queue, descriptor rings, free software buffers used when stopping
 257  * device.
 258  */
 259 static inline void
 260 rx_queue_clean(struct fm10k_rx_queue *q)
 261 {
 262         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 263         uint32_t i;
 264         PMD_INIT_FUNC_TRACE();
 265
 266         /* zero descriptor rings */
 267         for (i = 0; i < q->nb_desc; ++i)
 268                 q->hw_ring[i] = zero;
 269
 270         /* zero faked descriptors */
 271         for (i = 0; i < q->nb_fake_desc; ++i)
 272                 q->hw_ring[q->nb_desc + i] = zero;
 273
 274         /* vPMD driver has a different way of releasing mbufs. */
 275         if (q->rx_using_sse) {
 276                 fm10k_rx_queue_release_mbufs_vec(q);
 277                 return;
 278         }
 279
 280         /* free software buffers */
 281         for (i = 0; i < q->nb_desc; ++i) {
 282                 if (q->sw_ring[i]) {
 283                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 284                         q->sw_ring[i] = NULL;
 285                 }
 286         }
 287 }
 288
 289 /*
 290  * free all queue memory used when releasing the queue (i.e. configure)
 291  */
 292 static inline void
 293 rx_queue_free(struct fm10k_rx_queue *q)
 294 {
 295         PMD_INIT_FUNC_TRACE();
 296         if (q) {
 297                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 298                 rx_queue_clean(q);
 299                 if (q->sw_ring) {
 300                         rte_free(q->sw_ring);
 301                         q->sw_ring = NULL;
 302                 }
 303                 rte_free(q);
 304                 q = NULL;
 305         }
 306 }
 307
 308 /*
 309  * disable RX queue, wait unitl HW finished necessary flush operation
 310  */
 311 static inline int
 312 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 313 {
 314         uint32_t reg, i;
 315
 316         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 317         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 318                         reg & ~FM10K_RXQCTL_ENABLE);
 319
 320         /* Wait 100us at most */
 321         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 322                 rte_delay_us(1);
 323                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 324                 if (!(reg & FM10K_RXQCTL_ENABLE))
 325                         break;
 326         }
 327
 328         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 329                 return -1;
 330
 331         return 0;
 332 }
 333
 334 /*
 335  * reset queue to initial state, allocate software buffers used when starting
 336  * device
 337  */
 338 static inline void
 339 tx_queue_reset(struct fm10k_tx_queue *q)
 340 {
 341         PMD_INIT_FUNC_TRACE();
 342         q->last_free = 0;
 343         q->next_free = 0;
 344         q->nb_used = 0;
 345         q->nb_free = q->nb_desc - 1;
 346         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 347         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 348 }
 349
 350 /*
 351  * clean queue, descriptor rings, free software buffers used when stopping
 352  * device
 353  */
 354 static inline void
 355 tx_queue_clean(struct fm10k_tx_queue *q)
 356 {
 357         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 358         uint32_t i;
 359         PMD_INIT_FUNC_TRACE();
 360
 361         /* zero descriptor rings */
 362         for (i = 0; i < q->nb_desc; ++i)
 363                 q->hw_ring[i] = zero;
 364
 365         /* free software buffers */
 366         for (i = 0; i < q->nb_desc; ++i) {
 367                 if (q->sw_ring[i]) {
 368                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 369                         q->sw_ring[i] = NULL;
 370                 }
 371         }
 372 }
 373
 374 /*
 375  * free all queue memory used when releasing the queue (i.e. configure)
 376  */
 377 static inline void
 378 tx_queue_free(struct fm10k_tx_queue *q)
 379 {
 380         PMD_INIT_FUNC_TRACE();
 381         if (q) {
 382                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 383                 tx_queue_clean(q);
 384                 if (q->rs_tracker.list) {
 385                         rte_free(q->rs_tracker.list);
 386                         q->rs_tracker.list = NULL;
 387                 }
 388                 if (q->sw_ring) {
 389                         rte_free(q->sw_ring);
 390                         q->sw_ring = NULL;
 391                 }
 392                 rte_free(q);
 393                 q = NULL;
 394         }
 395 }
 396
 397 /*
 398  * disable TX queue, wait unitl HW finished necessary flush operation
 399  */
 400 static inline int
 401 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 402 {
 403         uint32_t reg, i;
 404
 405         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 406         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 407                         reg & ~FM10K_TXDCTL_ENABLE);
 408
 409         /* Wait 100us at most */
 410         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 411                 rte_delay_us(1);
 412                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 413                 if (!(reg & FM10K_TXDCTL_ENABLE))
 414                         break;
 415         }
 416
 417         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 418                 return -1;
 419
 420         return 0;
 421 }
 422
 423 static int
 424 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 425 {
 426         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 427         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 428         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 429         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 430
 431         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 432
 433         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 434                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 435                 return -EINVAL;
 436         }
 437
 438         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 439                 return 0;
 440
 441         if (hw->mac.type == fm10k_mac_vf) {
 442                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 443                 return -EINVAL;
 444         }
 445
 446         /* Check VMDQ queue pool number */
 447         if (vmdq_conf->nb_queue_pools >
 448                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 449                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 450                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 451                         vmdq_conf->nb_queue_pools);
 452                 return -EINVAL;
 453         }
 454
 455         return 0;
 456 }
 457
 458 static const struct fm10k_txq_ops def_txq_ops = {
 459         .reset = tx_queue_reset,
 460 };
 461
 462 static int
 463 fm10k_dev_configure(struct rte_eth_dev *dev)
 464 {
 465         int ret;
 466
 467         PMD_INIT_FUNC_TRACE();
 468
 469         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 470                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 471         /* multipe queue mode checking */
 472         ret  = fm10k_check_mq_mode(dev);
 473         if (ret != 0) {
 474                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 475                             ret);
 476                 return ret;
 477         }
 478
 479         return 0;
 480 }
 481
 482 /* fls = find last set bit = 32 minus the number of leading zeros */
 483 #ifndef fls
 484 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 485 #endif
 486
 487 static void
 488 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 489 {
 490         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 491         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 492         uint32_t i;
 493
 494         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 495
 496         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 497                 if (!vmdq_conf->pool_map[i].pools)
 498                         continue;
 499                 fm10k_mbx_lock(hw);
 500                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 501                 fm10k_mbx_unlock(hw);
 502         }
 503 }
 504
 505 static void
 506 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 507 {
 508         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 509
 510         /* Add default mac address */
 511         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 512                 MAIN_VSI_POOL_NUMBER);
 513 }
 514
 515 static void
 516 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 517 {
 518         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 519         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 520         uint32_t mrqc, *key, i, reta, j;
 521         uint64_t hf;
 522
 523 #define RSS_KEY_SIZE 40
 524         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 525                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 526                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 527                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 528                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 529                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 530         };
 531
 532         if (dev->data->nb_rx_queues == 1 ||
 533             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 534             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0) {
 535                 FM10K_WRITE_REG(hw, FM10K_MRQC(0), 0);
 536                 return;
 537         }
 538
 539         /* random key is rss_intel_key (default) or user provided (rss_key) */
 540         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 541                 key = (uint32_t *)rss_intel_key;
 542         else
 543                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 544
 545         /* Now fill our hash function seeds, 4 bytes at a time */
 546         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 547                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 548
 549         /*
 550          * Fill in redirection table
 551          * The byte-swap is needed because NIC registers are in
 552          * little-endian order.
 553          */
 554         reta = 0;
 555         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 556                 if (j == dev->data->nb_rx_queues)
 557                         j = 0;
 558                 reta = (reta << CHAR_BIT) | j;
 559                 if ((i & 3) == 3)
 560                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 561                                         rte_bswap32(reta));
 562         }
 563
 564         /*
 565          * Generate RSS hash based on packet types, TCP/UDP
 566          * port numbers and/or IPv4/v6 src and dst addresses
 567          */
 568         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 569         mrqc = 0;
 570         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 571         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 572         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 573         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 574         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 575         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 576         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 577         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 578         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 579
 580         if (mrqc == 0) {
 581                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 582                         "supported", hf);
 583                 return;
 584         }
 585
 586         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 587 }
 588
 589 static void
 590 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 591 {
 592         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 593         uint32_t i;
 594
 595         for (i = 0; i < nb_lport_new; i++) {
 596                 /* Set unicast mode by default. App can change
 597                  * to other mode in other API func.
 598                  */
 599                 fm10k_mbx_lock(hw);
 600                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 601                         FM10K_XCAST_MODE_NONE);
 602                 fm10k_mbx_unlock(hw);
 603         }
 604 }
 605
 606 static void
 607 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 608 {
 609         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 610         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 611         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 612         struct fm10k_macvlan_filter_info *macvlan;
 613         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 614         uint16_t nb_lport_new;
 615
 616         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 617         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 618
 619         fm10k_dev_rss_configure(dev);
 620
 621         /* only PF supports VMDQ */
 622         if (hw->mac.type != fm10k_mac_pf)
 623                 return;
 624
 625         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 626                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 627
 628         /* no pool number change, no need to update logic port and VLAN/MAC */
 629         if (macvlan->nb_queue_pools == nb_queue_pools)
 630                 return;
 631
 632         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 633         fm10k_dev_logic_port_update(dev, nb_lport_new);
 634
 635         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 636         memset(dev->data->mac_addrs, 0,
 637                 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 638         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
 639                 &dev->data->mac_addrs[0]);
 640         memset(macvlan, 0, sizeof(*macvlan));
 641         macvlan->nb_queue_pools = nb_queue_pools;
 642
 643         if (nb_queue_pools)
 644                 fm10k_dev_vmdq_rx_configure(dev);
 645         else
 646                 fm10k_dev_pf_main_vsi_reset(dev);
 647 }
 648
 649 static int
 650 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 651 {
 652         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 653         int i, ret;
 654         struct fm10k_tx_queue *txq;
 655         uint64_t base_addr;
 656         uint32_t size;
 657
 658         /* Disable TXINT to avoid possible interrupt */
 659         for (i = 0; i < hw->mac.max_queues; i++)
 660                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 661                                 3 << FM10K_TXINT_TIMER_SHIFT);
 662
 663         /* Setup TX queue */
 664         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 665                 txq = dev->data->tx_queues[i];
 666                 base_addr = txq->hw_ring_phys_addr;
 667                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 668
 669                 /* disable queue to avoid issues while updating state */
 670                 ret = tx_queue_disable(hw, i);
 671                 if (ret) {
 672                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 673                         return -1;
 674                 }
 675                 /* Enable use of FTAG bit in TX descriptor, PFVTCTL
 676                  * register is read-only for VF.
 677                  */
 678                 if (fm10k_check_ftag(dev->pci_dev->devargs)) {
 679                         if (hw->mac.type == fm10k_mac_pf) {
 680                                 FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i),
 681                                                 FM10K_PFVTCTL_FTAG_DESC_ENABLE);
 682                                 PMD_INIT_LOG(DEBUG, "FTAG mode is enabled");
 683                         } else {
 684                                 PMD_INIT_LOG(ERR, "VF FTAG is not supported.");
 685                                 return -ENOTSUP;
 686                         }
 687                 }
 688
 689                 /* set location and size for descriptor ring */
 690                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 691                                 base_addr & UINT64_LOWER_32BITS_MASK);
 692                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 693                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 694                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 695
 696                 /* assign default SGLORT for each TX queue */
 697                 FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map);
 698         }
 699
 700         /* set up vector or scalar TX function as appropriate */
 701         fm10k_set_tx_function(dev);
 702
 703         return 0;
 704 }
 705
 706 static int
 707 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 708 {
 709         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 710         struct fm10k_macvlan_filter_info *macvlan;
 711         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
 712         int i, ret;
 713         struct fm10k_rx_queue *rxq;
 714         uint64_t base_addr;
 715         uint32_t size;
 716         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 717         uint32_t logic_port = hw->mac.dglort_map;
 718         uint16_t buf_size;
 719         uint16_t queue_stride = 0;
 720
 721         /* enable RXINT for interrupt mode */
 722         i = 0;
 723         if (rte_intr_dp_is_en(intr_handle)) {
 724                 for (; i < dev->data->nb_rx_queues; i++) {
 725                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(dev, i));
 726                         if (hw->mac.type == fm10k_mac_pf)
 727                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, i)),
 728                                         FM10K_ITR_AUTOMASK |
 729                                         FM10K_ITR_MASK_CLEAR);
 730                         else
 731                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, i)),
 732                                         FM10K_ITR_AUTOMASK |
 733                                         FM10K_ITR_MASK_CLEAR);
 734                 }
 735         }
 736         /* Disable other RXINT to avoid possible interrupt */
 737         for (; i < hw->mac.max_queues; i++)
 738                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 739                         3 << FM10K_RXINT_TIMER_SHIFT);
 740
 741         /* Setup RX queues */
 742         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 743                 rxq = dev->data->rx_queues[i];
 744                 base_addr = rxq->hw_ring_phys_addr;
 745                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 746
 747                 /* disable queue to avoid issues while updating state */
 748                 ret = rx_queue_disable(hw, i);
 749                 if (ret) {
 750                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 751                         return -1;
 752                 }
 753
 754                 /* Setup the Base and Length of the Rx Descriptor Ring */
 755                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 756                                 base_addr & UINT64_LOWER_32BITS_MASK);
 757                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 758                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 759                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 760
 761                 /* Configure the Rx buffer size for one buff without split */
 762                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 763                         RTE_PKTMBUF_HEADROOM);
 764                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 765                  * reserved for this purpose, and the worst case could be 511B.
 766                  * But SRR reg assumes all buffers have the same size. In order
 767                  * to fill the gap, we'll have to consider the worst case and
 768                  * assume 512B is reserved. If we don't do so, it's possible
 769                  * for HW to overwrite data to next mbuf.
 770                  */
 771                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 772
 773                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 774                                 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
 775                                 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
 776
 777                 /* It adds dual VLAN length for supporting dual VLAN */
 778                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 779                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 780                         dev->data->dev_conf.rxmode.enable_scatter) {
 781                         uint32_t reg;
 782                         dev->data->scattered_rx = 1;
 783                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 784                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 785                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 786                 }
 787
 788                 /* Enable drop on empty, it's RO for VF */
 789                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 790                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 791
 792                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 793                 FM10K_WRITE_FLUSH(hw);
 794         }
 795
 796         /* Configure VMDQ/RSS if applicable */
 797         fm10k_dev_mq_rx_configure(dev);
 798
 799         /* Decide the best RX function */
 800         fm10k_set_rx_function(dev);
 801
 802         /* update RX_SGLORT for loopback suppress*/
 803         if (hw->mac.type != fm10k_mac_pf)
 804                 return 0;
 805         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 806         if (macvlan->nb_queue_pools)
 807                 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
 808         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 809                 if (i && queue_stride && !(i % queue_stride))
 810                         logic_port++;
 811                 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
 812         }
 813
 814         return 0;
 815 }
 816
 817 static int
 818 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 819 {
 820         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 821         int err = -1;
 822         uint32_t reg;
 823         struct fm10k_rx_queue *rxq;
 824
 825         PMD_INIT_FUNC_TRACE();
 826
 827         if (rx_queue_id < dev->data->nb_rx_queues) {
 828                 rxq = dev->data->rx_queues[rx_queue_id];
 829                 err = rx_queue_reset(rxq);
 830                 if (err == -ENOMEM) {
 831                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 832                         return err;
 833                 } else if (err == -EINVAL) {
 834                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 835                                 " %d", err);
 836                         return err;
 837                 }
 838
 839                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 840                  * Note: this must be done AFTER the queue is enabled on real
 841                  * hardware, but BEFORE the queue is enabled when using the
 842                  * emulation platform. Do it in both places for now and remove
 843                  * this comment and the following two register writes when the
 844                  * emulation platform is no longer being used.
 845                  */
 846                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 847                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 848
 849                 /* Set PF ownership flag for PF devices */
 850                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 851                 if (hw->mac.type == fm10k_mac_pf)
 852                         reg |= FM10K_RXQCTL_PF;
 853                 reg |= FM10K_RXQCTL_ENABLE;
 854                 /* enable RX queue */
 855                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 856                 FM10K_WRITE_FLUSH(hw);
 857
 858                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 859                  * Note: this must be done AFTER the queue is enabled
 860                  */
 861                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 862                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 863                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 864         }
 865
 866         return err;
 867 }
 868
 869 static int
 870 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 871 {
 872         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 873
 874         PMD_INIT_FUNC_TRACE();
 875
 876         if (rx_queue_id < dev->data->nb_rx_queues) {
 877                 /* Disable RX queue */
 878                 rx_queue_disable(hw, rx_queue_id);
 879
 880                 /* Free mbuf and clean HW ring */
 881                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 882                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 883         }
 884
 885         return 0;
 886 }
 887
 888 static int
 889 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 890 {
 891         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 892         /** @todo - this should be defined in the shared code */
 893 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 894         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 895         int err = 0;
 896
 897         PMD_INIT_FUNC_TRACE();
 898
 899         if (tx_queue_id < dev->data->nb_tx_queues) {
 900                 struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 901
 902                 q->ops->reset(q);
 903
 904                 /* reset head and tail pointers */
 905                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 906                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 907
 908                 /* enable TX queue */
 909                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 910                                         FM10K_TXDCTL_ENABLE | txdctl);
 911                 FM10K_WRITE_FLUSH(hw);
 912                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 913         } else
 914                 err = -1;
 915
 916         return err;
 917 }
 918
 919 static int
 920 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 921 {
 922         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 923
 924         PMD_INIT_FUNC_TRACE();
 925
 926         if (tx_queue_id < dev->data->nb_tx_queues) {
 927                 tx_queue_disable(hw, tx_queue_id);
 928                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 929                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 930         }
 931
 932         return 0;
 933 }
 934
 935 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 936 {
 937         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 938                 != FM10K_DGLORTMAP_NONE);
 939 }
 940
 941 static void
 942 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 943 {
 944         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 945         int status;
 946
 947         PMD_INIT_FUNC_TRACE();
 948
 949         /* Return if it didn't acquire valid glort range */
 950         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 951                 return;
 952
 953         fm10k_mbx_lock(hw);
 954         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 955                                 FM10K_XCAST_MODE_PROMISC);
 956         fm10k_mbx_unlock(hw);
 957
 958         if (status != FM10K_SUCCESS)
 959                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 960 }
 961
 962 static void
 963 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 964 {
 965         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 966         uint8_t mode;
 967         int status;
 968
 969         PMD_INIT_FUNC_TRACE();
 970
 971         /* Return if it didn't acquire valid glort range */
 972         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 973                 return;
 974
 975         if (dev->data->all_multicast == 1)
 976                 mode = FM10K_XCAST_MODE_ALLMULTI;
 977         else
 978                 mode = FM10K_XCAST_MODE_NONE;
 979
 980         fm10k_mbx_lock(hw);
 981         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 982                                 mode);
 983         fm10k_mbx_unlock(hw);
 984
 985         if (status != FM10K_SUCCESS)
 986                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 987 }
 988
 989 static void
 990 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 991 {
 992         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 993         int status;
 994
 995         PMD_INIT_FUNC_TRACE();
 996
 997         /* Return if it didn't acquire valid glort range */
 998         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 999                 return;
1000
1001         /* If promiscuous mode is enabled, it doesn't make sense to enable
1002          * allmulticast and disable promiscuous since fm10k only can select
1003          * one of the modes.
1004          */
1005         if (dev->data->promiscuous) {
1006                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
1007                         "needn't enable allmulticast");
1008                 return;
1009         }
1010
1011         fm10k_mbx_lock(hw);
1012         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1013                                 FM10K_XCAST_MODE_ALLMULTI);
1014         fm10k_mbx_unlock(hw);
1015
1016         if (status != FM10K_SUCCESS)
1017                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
1018 }
1019
1020 static void
1021 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
1022 {
1023         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1024         int status;
1025
1026         PMD_INIT_FUNC_TRACE();
1027
1028         /* Return if it didn't acquire valid glort range */
1029         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1030                 return;
1031
1032         if (dev->data->promiscuous) {
1033                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1034                         "since promisc mode is enabled");
1035                 return;
1036         }
1037
1038         fm10k_mbx_lock(hw);
1039         /* Change mode to unicast mode */
1040         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1041                                 FM10K_XCAST_MODE_NONE);
1042         fm10k_mbx_unlock(hw);
1043
1044         if (status != FM10K_SUCCESS)
1045                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1046 }
1047
1048 static void
1049 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1050 {
1051         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1052         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1053         uint16_t nb_queue_pools;
1054         struct fm10k_macvlan_filter_info *macvlan;
1055
1056         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1057         nb_queue_pools = macvlan->nb_queue_pools;
1058         pool_len = nb_queue_pools ? fls(nb_queue_pools - 1) : 0;
1059         rss_len = fls(dev->data->nb_rx_queues - 1) - pool_len;
1060
1061         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1062         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1063         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1064                         hw->mac.dglort_map;
1065         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1066         /* Configure VMDQ/RSS DGlort Decoder */
1067         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1068
1069         /* Flow Director configurations, only queue number is valid. */
1070         dglortdec = fls(dev->data->nb_rx_queues - 1);
1071         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1072                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1073         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1074         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1075
1076         /* Invalidate all other GLORT entries */
1077         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1078                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1079                                 FM10K_DGLORTMAP_NONE);
1080 }
1081
1082 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1083 static int
1084 fm10k_dev_start(struct rte_eth_dev *dev)
1085 {
1086         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1087         int i, diag;
1088
1089         PMD_INIT_FUNC_TRACE();
1090
1091         /* stop, init, then start the hw */
1092         diag = fm10k_stop_hw(hw);
1093         if (diag != FM10K_SUCCESS) {
1094                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1095                 return -EIO;
1096         }
1097
1098         diag = fm10k_init_hw(hw);
1099         if (diag != FM10K_SUCCESS) {
1100                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1101                 return -EIO;
1102         }
1103
1104         diag = fm10k_start_hw(hw);
1105         if (diag != FM10K_SUCCESS) {
1106                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1107                 return -EIO;
1108         }
1109
1110         diag = fm10k_dev_tx_init(dev);
1111         if (diag) {
1112                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1113                 return diag;
1114         }
1115
1116         if (fm10k_dev_rxq_interrupt_setup(dev))
1117                 return -EIO;
1118
1119         diag = fm10k_dev_rx_init(dev);
1120         if (diag) {
1121                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1122                 return diag;
1123         }
1124
1125         if (hw->mac.type == fm10k_mac_pf)
1126                 fm10k_dev_dglort_map_configure(dev);
1127
1128         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1129                 struct fm10k_rx_queue *rxq;
1130                 rxq = dev->data->rx_queues[i];
1131
1132                 if (rxq->rx_deferred_start)
1133                         continue;
1134                 diag = fm10k_dev_rx_queue_start(dev, i);
1135                 if (diag != 0) {
1136                         int j;
1137                         for (j = 0; j < i; ++j)
1138                                 rx_queue_clean(dev->data->rx_queues[j]);
1139                         return diag;
1140                 }
1141         }
1142
1143         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1144                 struct fm10k_tx_queue *txq;
1145                 txq = dev->data->tx_queues[i];
1146
1147                 if (txq->tx_deferred_start)
1148                         continue;
1149                 diag = fm10k_dev_tx_queue_start(dev, i);
1150                 if (diag != 0) {
1151                         int j;
1152                         for (j = 0; j < i; ++j)
1153                                 tx_queue_clean(dev->data->tx_queues[j]);
1154                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1155                                 rx_queue_clean(dev->data->rx_queues[j]);
1156                         return diag;
1157                 }
1158         }
1159
1160         /* Update default vlan when not in VMDQ mode */
1161         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1162                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1163
1164         return 0;
1165 }
1166
1167 static void
1168 fm10k_dev_stop(struct rte_eth_dev *dev)
1169 {
1170         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1171         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1172         int i;
1173
1174         PMD_INIT_FUNC_TRACE();
1175
1176         if (dev->data->tx_queues)
1177                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1178                         fm10k_dev_tx_queue_stop(dev, i);
1179
1180         if (dev->data->rx_queues)
1181                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1182                         fm10k_dev_rx_queue_stop(dev, i);
1183
1184         /* Disable datapath event */
1185         if (rte_intr_dp_is_en(intr_handle)) {
1186                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1187                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1188                                 3 << FM10K_RXINT_TIMER_SHIFT);
1189                         if (hw->mac.type == fm10k_mac_pf)
1190                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, i)),
1191                                         FM10K_ITR_MASK_SET);
1192                         else
1193                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, i)),
1194                                         FM10K_ITR_MASK_SET);
1195                 }
1196         }
1197         /* Clean datapath event and queue/vec mapping */
1198         rte_intr_efd_disable(intr_handle);
1199         rte_free(intr_handle->intr_vec);
1200         intr_handle->intr_vec = NULL;
1201 }
1202
1203 static void
1204 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1205 {
1206         int i;
1207
1208         PMD_INIT_FUNC_TRACE();
1209
1210         if (dev->data->tx_queues) {
1211                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1212                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1213
1214                         tx_queue_free(txq);
1215                 }
1216         }
1217
1218         if (dev->data->rx_queues) {
1219                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1220                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1221         }
1222 }
1223
1224 static void
1225 fm10k_dev_close(struct rte_eth_dev *dev)
1226 {
1227         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1228
1229         PMD_INIT_FUNC_TRACE();
1230
1231         fm10k_mbx_lock(hw);
1232         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1233                 MAX_LPORT_NUM, false);
1234         fm10k_mbx_unlock(hw);
1235
1236         /* Stop mailbox service first */
1237         fm10k_close_mbx_service(hw);
1238         fm10k_dev_stop(dev);
1239         fm10k_dev_queue_release(dev);
1240         fm10k_stop_hw(hw);
1241 }
1242
1243 static int
1244 fm10k_link_update(struct rte_eth_dev *dev,
1245         __rte_unused int wait_to_complete)
1246 {
1247         PMD_INIT_FUNC_TRACE();
1248
1249         /* The host-interface link is always up.  The speed is ~50Gbps per Gen3
1250          * x8 PCIe interface. For now, we leave the speed undefined since there
1251          * is no 50Gbps Ethernet. */
1252         dev->data->dev_link.link_speed  = 0;
1253         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1254         dev->data->dev_link.link_status = ETH_LINK_UP;
1255
1256         return 0;
1257 }
1258
1259 static int fm10k_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1260         struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit)
1261 {
1262         unsigned i, q;
1263         unsigned count = 0;
1264
1265         if (xstats_names != NULL) {
1266                 /* Note: limit checked in rte_eth_xstats_names() */
1267
1268                 /* Global stats */
1269                 for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1270                         snprintf(xstats_names[count].name,
1271                                 sizeof(xstats_names[count].name),
1272                                 "%s", fm10k_hw_stats_strings[count].name);
1273                         count++;
1274                 }
1275
1276                 /* PF queue stats */
1277                 for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1278                         for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1279                                 snprintf(xstats_names[count].name,
1280                                         sizeof(xstats_names[count].name),
1281                                         "rx_q%u_%s", q,
1282                                         fm10k_hw_stats_rx_q_strings[i].name);
1283                                 count++;
1284                         }
1285                         for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1286                                 snprintf(xstats_names[count].name,
1287                                         sizeof(xstats_names[count].name),
1288                                         "tx_q%u_%s", q,
1289                                         fm10k_hw_stats_tx_q_strings[i].name);
1290                                 count++;
1291                         }
1292                 }
1293         }
1294         return FM10K_NB_XSTATS;
1295 }
1296
1297 static int
1298 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1299                  unsigned n)
1300 {
1301         struct fm10k_hw_stats *hw_stats =
1302                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1303         unsigned i, q, count = 0;
1304
1305         if (n < FM10K_NB_XSTATS)
1306                 return FM10K_NB_XSTATS;
1307
1308         /* Global stats */
1309         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1310                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1311                         fm10k_hw_stats_strings[count].offset);
1312                 count++;
1313         }
1314
1315         /* PF queue stats */
1316         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1317                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1318                         xstats[count].value =
1319                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1320                                 fm10k_hw_stats_rx_q_strings[i].offset);
1321                         count++;
1322                 }
1323                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1324                         xstats[count].value =
1325                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1326                                 fm10k_hw_stats_tx_q_strings[i].offset);
1327                         count++;
1328                 }
1329         }
1330
1331         return FM10K_NB_XSTATS;
1332 }
1333
1334 static void
1335 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1336 {
1337         uint64_t ipackets, opackets, ibytes, obytes;
1338         struct fm10k_hw *hw =
1339                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1340         struct fm10k_hw_stats *hw_stats =
1341                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1342         int i;
1343
1344         PMD_INIT_FUNC_TRACE();
1345
1346         fm10k_update_hw_stats(hw, hw_stats);
1347
1348         ipackets = opackets = ibytes = obytes = 0;
1349         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1350                 (i < hw->mac.max_queues); ++i) {
1351                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1352                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1353                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1354                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1355                 ipackets += stats->q_ipackets[i];
1356                 opackets += stats->q_opackets[i];
1357                 ibytes   += stats->q_ibytes[i];
1358                 obytes   += stats->q_obytes[i];
1359         }
1360         stats->ipackets = ipackets;
1361         stats->opackets = opackets;
1362         stats->ibytes = ibytes;
1363         stats->obytes = obytes;
1364 }
1365
1366 static void
1367 fm10k_stats_reset(struct rte_eth_dev *dev)
1368 {
1369         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1370         struct fm10k_hw_stats *hw_stats =
1371                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1372
1373         PMD_INIT_FUNC_TRACE();
1374
1375         memset(hw_stats, 0, sizeof(*hw_stats));
1376         fm10k_rebind_hw_stats(hw, hw_stats);
1377 }
1378
1379 static void
1380 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1381         struct rte_eth_dev_info *dev_info)
1382 {
1383         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1384
1385         PMD_INIT_FUNC_TRACE();
1386
1387         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1388         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1389         dev_info->max_rx_queues      = hw->mac.max_queues;
1390         dev_info->max_tx_queues      = hw->mac.max_queues;
1391         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1392         dev_info->max_hash_mac_addrs = 0;
1393         dev_info->max_vfs            = dev->pci_dev->max_vfs;
1394         dev_info->vmdq_pool_base     = 0;
1395         dev_info->vmdq_queue_base    = 0;
1396         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1397         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1398         dev_info->rx_offload_capa =
1399                 DEV_RX_OFFLOAD_VLAN_STRIP |
1400                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1401                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1402                 DEV_RX_OFFLOAD_TCP_CKSUM;
1403         dev_info->tx_offload_capa =
1404                 DEV_TX_OFFLOAD_VLAN_INSERT |
1405                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1406                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1407                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1408                 DEV_TX_OFFLOAD_TCP_TSO;
1409
1410         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1411         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1412
1413         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1414                 .rx_thresh = {
1415                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1416                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1417                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1418                 },
1419                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1420                 .rx_drop_en = 0,
1421         };
1422
1423         dev_info->default_txconf = (struct rte_eth_txconf) {
1424                 .tx_thresh = {
1425                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1426                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1427                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1428                 },
1429                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1430                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1431                 .txq_flags = FM10K_SIMPLE_TX_FLAG,
1432         };
1433
1434         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1435                 .nb_max = FM10K_MAX_RX_DESC,
1436                 .nb_min = FM10K_MIN_RX_DESC,
1437                 .nb_align = FM10K_MULT_RX_DESC,
1438         };
1439
1440         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1441                 .nb_max = FM10K_MAX_TX_DESC,
1442                 .nb_min = FM10K_MIN_TX_DESC,
1443                 .nb_align = FM10K_MULT_TX_DESC,
1444         };
1445
1446         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
1447                         ETH_LINK_SPEED_10G | ETH_LINK_SPEED_25G |
1448                         ETH_LINK_SPEED_40G | ETH_LINK_SPEED_100G;
1449 }
1450
1451 #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
1452 static const uint32_t *
1453 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1454 {
1455         if (dev->rx_pkt_burst == fm10k_recv_pkts ||
1456             dev->rx_pkt_burst == fm10k_recv_scattered_pkts) {
1457                 static uint32_t ptypes[] = {
1458                         /* refers to rx_desc_to_ol_flags() */
1459                         RTE_PTYPE_L2_ETHER,
1460                         RTE_PTYPE_L3_IPV4,
1461                         RTE_PTYPE_L3_IPV4_EXT,
1462                         RTE_PTYPE_L3_IPV6,
1463                         RTE_PTYPE_L3_IPV6_EXT,
1464                         RTE_PTYPE_L4_TCP,
1465                         RTE_PTYPE_L4_UDP,
1466                         RTE_PTYPE_UNKNOWN
1467                 };
1468
1469                 return ptypes;
1470         } else if (dev->rx_pkt_burst == fm10k_recv_pkts_vec ||
1471                    dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec) {
1472                 static uint32_t ptypes_vec[] = {
1473                         /* refers to fm10k_desc_to_pktype_v() */
1474                         RTE_PTYPE_L3_IPV4,
1475                         RTE_PTYPE_L3_IPV4_EXT,
1476                         RTE_PTYPE_L3_IPV6,
1477                         RTE_PTYPE_L3_IPV6_EXT,
1478                         RTE_PTYPE_L4_TCP,
1479                         RTE_PTYPE_L4_UDP,
1480                         RTE_PTYPE_TUNNEL_GENEVE,
1481                         RTE_PTYPE_TUNNEL_NVGRE,
1482                         RTE_PTYPE_TUNNEL_VXLAN,
1483                         RTE_PTYPE_TUNNEL_GRE,
1484                         RTE_PTYPE_UNKNOWN
1485                 };
1486
1487                 return ptypes_vec;
1488         }
1489
1490         return NULL;
1491 }
1492 #else
1493 static const uint32_t *
1494 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1495 {
1496         return NULL;
1497 }
1498 #endif
1499
1500 static int
1501 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1502 {
1503         s32 result;
1504         uint16_t mac_num = 0;
1505         uint32_t vid_idx, vid_bit, mac_index;
1506         struct fm10k_hw *hw;
1507         struct fm10k_macvlan_filter_info *macvlan;
1508         struct rte_eth_dev_data *data = dev->data;
1509
1510         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1511         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1512
1513         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1514                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1515                 return -EINVAL;
1516         }
1517
1518         if (vlan_id > ETH_VLAN_ID_MAX) {
1519                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1520                 return -EINVAL;
1521         }
1522
1523         vid_idx = FM10K_VFTA_IDX(vlan_id);
1524         vid_bit = FM10K_VFTA_BIT(vlan_id);
1525         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1526         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1527                 return 0;
1528         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1529         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1530                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1531                         "in the VLAN filter table");
1532                 return -EINVAL;
1533         }
1534
1535         fm10k_mbx_lock(hw);
1536         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1537         fm10k_mbx_unlock(hw);
1538         if (result != FM10K_SUCCESS) {
1539                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1540                 return -EIO;
1541         }
1542
1543         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1544                         (result == FM10K_SUCCESS); mac_index++) {
1545                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1546                         continue;
1547                 if (mac_num > macvlan->mac_num - 1) {
1548                         PMD_INIT_LOG(ERR, "MAC address number "
1549                                         "not match");
1550                         break;
1551                 }
1552                 fm10k_mbx_lock(hw);
1553                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1554                         data->mac_addrs[mac_index].addr_bytes,
1555                         vlan_id, on, 0);
1556                 fm10k_mbx_unlock(hw);
1557                 mac_num++;
1558         }
1559         if (result != FM10K_SUCCESS) {
1560                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1561                 return -EIO;
1562         }
1563
1564         if (on) {
1565                 macvlan->vlan_num++;
1566                 macvlan->vfta[vid_idx] |= vid_bit;
1567         } else {
1568                 macvlan->vlan_num--;
1569                 macvlan->vfta[vid_idx] &= ~vid_bit;
1570         }
1571         return 0;
1572 }
1573
1574 static void
1575 fm10k_vlan_offload_set(__rte_unused struct rte_eth_dev *dev, int mask)
1576 {
1577         if (mask & ETH_VLAN_STRIP_MASK) {
1578                 if (!dev->data->dev_conf.rxmode.hw_vlan_strip)
1579                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1580                                         "always on in fm10k");
1581         }
1582
1583         if (mask & ETH_VLAN_EXTEND_MASK) {
1584                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1585                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1586                                         "supported in fm10k");
1587         }
1588
1589         if (mask & ETH_VLAN_FILTER_MASK) {
1590                 if (!dev->data->dev_conf.rxmode.hw_vlan_filter)
1591                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1592         }
1593 }
1594
1595 /* Add/Remove a MAC address, and update filters to main VSI */
1596 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1597                 const u8 *mac, bool add, uint32_t pool)
1598 {
1599         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1600         struct fm10k_macvlan_filter_info *macvlan;
1601         uint32_t i, j, k;
1602
1603         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1604
1605         if (pool != MAIN_VSI_POOL_NUMBER) {
1606                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1607                         "mac to pool %u", pool);
1608                 return;
1609         }
1610         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1611                 if (!macvlan->vfta[j])
1612                         continue;
1613                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1614                         if (!(macvlan->vfta[j] & (1 << k)))
1615                                 continue;
1616                         if (i + 1 > macvlan->vlan_num) {
1617                                 PMD_INIT_LOG(ERR, "vlan number not match");
1618                                 return;
1619                         }
1620                         fm10k_mbx_lock(hw);
1621                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1622                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1623                         fm10k_mbx_unlock(hw);
1624                         i++;
1625                 }
1626         }
1627 }
1628
1629 /* Add/Remove a MAC address, and update filters to VMDQ */
1630 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1631                 const u8 *mac, bool add, uint32_t pool)
1632 {
1633         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1634         struct fm10k_macvlan_filter_info *macvlan;
1635         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1636         uint32_t i;
1637
1638         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1639         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1640
1641         if (pool > macvlan->nb_queue_pools) {
1642                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1643                         " Max pool is %u",
1644                         pool, macvlan->nb_queue_pools);
1645                 return;
1646         }
1647         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1648                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1649                         continue;
1650                 fm10k_mbx_lock(hw);
1651                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1652                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1653                 fm10k_mbx_unlock(hw);
1654         }
1655 }
1656
1657 /* Add/Remove a MAC address, and update filters */
1658 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1659                 const u8 *mac, bool add, uint32_t pool)
1660 {
1661         struct fm10k_macvlan_filter_info *macvlan;
1662
1663         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1664
1665         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1666                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1667         else
1668                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1669
1670         if (add)
1671                 macvlan->mac_num++;
1672         else
1673                 macvlan->mac_num--;
1674 }
1675
1676 /* Add a MAC address, and update filters */
1677 static void
1678 fm10k_macaddr_add(struct rte_eth_dev *dev,
1679                 struct ether_addr *mac_addr,
1680                 uint32_t index,
1681                 uint32_t pool)
1682 {
1683         struct fm10k_macvlan_filter_info *macvlan;
1684
1685         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1686         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1687         macvlan->mac_vmdq_id[index] = pool;
1688 }
1689
1690 /* Remove a MAC address, and update filters */
1691 static void
1692 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1693 {
1694         struct rte_eth_dev_data *data = dev->data;
1695         struct fm10k_macvlan_filter_info *macvlan;
1696
1697         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1698         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1699                         FALSE, macvlan->mac_vmdq_id[index]);
1700         macvlan->mac_vmdq_id[index] = 0;
1701 }
1702
1703 static inline int
1704 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1705 {
1706         if ((request < min) || (request > max) || ((request % mult) != 0))
1707                 return -1;
1708         else
1709                 return 0;
1710 }
1711
1712
1713 static inline int
1714 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1715 {
1716         if ((request < min) || (request > max) || ((div % request) != 0))
1717                 return -1;
1718         else
1719                 return 0;
1720 }
1721
1722 static inline int
1723 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1724 {
1725         uint16_t rx_free_thresh;
1726
1727         if (conf->rx_free_thresh == 0)
1728                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1729         else
1730                 rx_free_thresh = conf->rx_free_thresh;
1731
1732         /* make sure the requested threshold satisfies the constraints */
1733         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1734                         FM10K_RX_FREE_THRESH_MAX(q),
1735                         FM10K_RX_FREE_THRESH_DIV(q),
1736                         rx_free_thresh)) {
1737                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1738                         "less than or equal to %u, "
1739                         "greater than or equal to %u, "
1740                         "and a divisor of %u",
1741                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1742                         FM10K_RX_FREE_THRESH_MIN(q),
1743                         FM10K_RX_FREE_THRESH_DIV(q));
1744                 return -EINVAL;
1745         }
1746
1747         q->alloc_thresh = rx_free_thresh;
1748         q->drop_en = conf->rx_drop_en;
1749         q->rx_deferred_start = conf->rx_deferred_start;
1750
1751         return 0;
1752 }
1753
1754 /*
1755  * Hardware requires specific alignment for Rx packet buffers. At
1756  * least one of the following two conditions must be satisfied.
1757  *  1. Address is 512B aligned
1758  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1759  *
1760  * As such, the driver may need to adjust the DMA address within the
1761  * buffer by up to 512B.
1762  *
1763  * return 1 if the element size is valid, otherwise return 0.
1764  */
1765 static int
1766 mempool_element_size_valid(struct rte_mempool *mp)
1767 {
1768         uint32_t min_size;
1769
1770         /* elt_size includes mbuf header and headroom */
1771         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1772                         RTE_PKTMBUF_HEADROOM;
1773
1774         /* account for up to 512B of alignment */
1775         min_size -= FM10K_RX_DATABUF_ALIGN;
1776
1777         /* sanity check for overflow */
1778         if (min_size > mp->elt_size)
1779                 return 0;
1780
1781         /* size is valid */
1782         return 1;
1783 }
1784
1785 static int
1786 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1787         uint16_t nb_desc, unsigned int socket_id,
1788         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1789 {
1790         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1791         struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev);
1792         struct fm10k_rx_queue *q;
1793         const struct rte_memzone *mz;
1794
1795         PMD_INIT_FUNC_TRACE();
1796
1797         /* make sure the mempool element size can account for alignment. */
1798         if (!mempool_element_size_valid(mp)) {
1799                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1800                 return -EINVAL;
1801         }
1802
1803         /* make sure a valid number of descriptors have been requested */
1804         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1805                                 FM10K_MULT_RX_DESC, nb_desc)) {
1806                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1807                         "less than or equal to %"PRIu32", "
1808                         "greater than or equal to %u, "
1809                         "and a multiple of %u",
1810                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1811                         FM10K_MULT_RX_DESC);
1812                 return -EINVAL;
1813         }
1814
1815         /*
1816          * if this queue existed already, free the associated memory. The
1817          * queue cannot be reused in case we need to allocate memory on
1818          * different socket than was previously used.
1819          */
1820         if (dev->data->rx_queues[queue_id] != NULL) {
1821                 rx_queue_free(dev->data->rx_queues[queue_id]);
1822                 dev->data->rx_queues[queue_id] = NULL;
1823         }
1824
1825         /* allocate memory for the queue structure */
1826         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1827                                 socket_id);
1828         if (q == NULL) {
1829                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1830                 return -ENOMEM;
1831         }
1832
1833         /* setup queue */
1834         q->mp = mp;
1835         q->nb_desc = nb_desc;
1836         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1837         q->port_id = dev->data->port_id;
1838         q->queue_id = queue_id;
1839         q->tail_ptr = (volatile uint32_t *)
1840                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1841         if (handle_rxconf(q, conf))
1842                 return -EINVAL;
1843
1844         /* allocate memory for the software ring */
1845         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1846                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1847                         RTE_CACHE_LINE_SIZE, socket_id);
1848         if (q->sw_ring == NULL) {
1849                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1850                 rte_free(q);
1851                 return -ENOMEM;
1852         }
1853
1854         /*
1855          * allocate memory for the hardware descriptor ring. A memzone large
1856          * enough to hold the maximum ring size is requested to allow for
1857          * resizing in later calls to the queue setup function.
1858          */
1859         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1860                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1861                                       socket_id);
1862         if (mz == NULL) {
1863                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1864                 rte_free(q->sw_ring);
1865                 rte_free(q);
1866                 return -ENOMEM;
1867         }
1868         q->hw_ring = mz->addr;
1869         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1870
1871         /* Check if number of descs satisfied Vector requirement */
1872         if (!rte_is_power_of_2(nb_desc)) {
1873                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1874                                     "preconditions - canceling the feature for "
1875                                     "the whole port[%d]",
1876                              q->queue_id, q->port_id);
1877                 dev_info->rx_vec_allowed = false;
1878         } else
1879                 fm10k_rxq_vec_setup(q);
1880
1881         dev->data->rx_queues[queue_id] = q;
1882         return 0;
1883 }
1884
1885 static void
1886 fm10k_rx_queue_release(void *queue)
1887 {
1888         PMD_INIT_FUNC_TRACE();
1889
1890         rx_queue_free(queue);
1891 }
1892
1893 static inline int
1894 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1895 {
1896         uint16_t tx_free_thresh;
1897         uint16_t tx_rs_thresh;
1898
1899         /* constraint MACROs require that tx_free_thresh is configured
1900          * before tx_rs_thresh */
1901         if (conf->tx_free_thresh == 0)
1902                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1903         else
1904                 tx_free_thresh = conf->tx_free_thresh;
1905
1906         /* make sure the requested threshold satisfies the constraints */
1907         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1908                         FM10K_TX_FREE_THRESH_MAX(q),
1909                         FM10K_TX_FREE_THRESH_DIV(q),
1910                         tx_free_thresh)) {
1911                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1912                         "less than or equal to %u, "
1913                         "greater than or equal to %u, "
1914                         "and a divisor of %u",
1915                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1916                         FM10K_TX_FREE_THRESH_MIN(q),
1917                         FM10K_TX_FREE_THRESH_DIV(q));
1918                 return -EINVAL;
1919         }
1920
1921         q->free_thresh = tx_free_thresh;
1922
1923         if (conf->tx_rs_thresh == 0)
1924                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1925         else
1926                 tx_rs_thresh = conf->tx_rs_thresh;
1927
1928         q->tx_deferred_start = conf->tx_deferred_start;
1929
1930         /* make sure the requested threshold satisfies the constraints */
1931         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1932                         FM10K_TX_RS_THRESH_MAX(q),
1933                         FM10K_TX_RS_THRESH_DIV(q),
1934                         tx_rs_thresh)) {
1935                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1936                         "less than or equal to %u, "
1937                         "greater than or equal to %u, "
1938                         "and a divisor of %u",
1939                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1940                         FM10K_TX_RS_THRESH_MIN(q),
1941                         FM10K_TX_RS_THRESH_DIV(q));
1942                 return -EINVAL;
1943         }
1944
1945         q->rs_thresh = tx_rs_thresh;
1946
1947         return 0;
1948 }
1949
1950 static int
1951 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1952         uint16_t nb_desc, unsigned int socket_id,
1953         const struct rte_eth_txconf *conf)
1954 {
1955         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1956         struct fm10k_tx_queue *q;
1957         const struct rte_memzone *mz;
1958
1959         PMD_INIT_FUNC_TRACE();
1960
1961         /* make sure a valid number of descriptors have been requested */
1962         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1963                                 FM10K_MULT_TX_DESC, nb_desc)) {
1964                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1965                         "less than or equal to %"PRIu32", "
1966                         "greater than or equal to %u, "
1967                         "and a multiple of %u",
1968                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1969                         FM10K_MULT_TX_DESC);
1970                 return -EINVAL;
1971         }
1972
1973         /*
1974          * if this queue existed already, free the associated memory. The
1975          * queue cannot be reused in case we need to allocate memory on
1976          * different socket than was previously used.
1977          */
1978         if (dev->data->tx_queues[queue_id] != NULL) {
1979                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
1980
1981                 tx_queue_free(txq);
1982                 dev->data->tx_queues[queue_id] = NULL;
1983         }
1984
1985         /* allocate memory for the queue structure */
1986         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1987                                 socket_id);
1988         if (q == NULL) {
1989                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1990                 return -ENOMEM;
1991         }
1992
1993         /* setup queue */
1994         q->nb_desc = nb_desc;
1995         q->port_id = dev->data->port_id;
1996         q->queue_id = queue_id;
1997         q->txq_flags = conf->txq_flags;
1998         q->ops = &def_txq_ops;
1999         q->tail_ptr = (volatile uint32_t *)
2000                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
2001         if (handle_txconf(q, conf))
2002                 return -EINVAL;
2003
2004         /* allocate memory for the software ring */
2005         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
2006                                         nb_desc * sizeof(struct rte_mbuf *),
2007                                         RTE_CACHE_LINE_SIZE, socket_id);
2008         if (q->sw_ring == NULL) {
2009                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
2010                 rte_free(q);
2011                 return -ENOMEM;
2012         }
2013
2014         /*
2015          * allocate memory for the hardware descriptor ring. A memzone large
2016          * enough to hold the maximum ring size is requested to allow for
2017          * resizing in later calls to the queue setup function.
2018          */
2019         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
2020                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
2021                                       socket_id);
2022         if (mz == NULL) {
2023                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
2024                 rte_free(q->sw_ring);
2025                 rte_free(q);
2026                 return -ENOMEM;
2027         }
2028         q->hw_ring = mz->addr;
2029         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
2030
2031         /*
2032          * allocate memory for the RS bit tracker. Enough slots to hold the
2033          * descriptor index for each RS bit needing to be set are required.
2034          */
2035         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
2036                                 ((nb_desc + 1) / q->rs_thresh) *
2037                                 sizeof(uint16_t),
2038                                 RTE_CACHE_LINE_SIZE, socket_id);
2039         if (q->rs_tracker.list == NULL) {
2040                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
2041                 rte_free(q->sw_ring);
2042                 rte_free(q);
2043                 return -ENOMEM;
2044         }
2045
2046         dev->data->tx_queues[queue_id] = q;
2047         return 0;
2048 }
2049
2050 static void
2051 fm10k_tx_queue_release(void *queue)
2052 {
2053         struct fm10k_tx_queue *q = queue;
2054         PMD_INIT_FUNC_TRACE();
2055
2056         tx_queue_free(q);
2057 }
2058
2059 static int
2060 fm10k_reta_update(struct rte_eth_dev *dev,
2061                         struct rte_eth_rss_reta_entry64 *reta_conf,
2062                         uint16_t reta_size)
2063 {
2064         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2065         uint16_t i, j, idx, shift;
2066         uint8_t mask;
2067         uint32_t reta;
2068
2069         PMD_INIT_FUNC_TRACE();
2070
2071         if (reta_size > FM10K_MAX_RSS_INDICES) {
2072                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2073                         "(%d) doesn't match the number hardware can supported "
2074                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2075                 return -EINVAL;
2076         }
2077
2078         /*
2079          * Update Redirection Table RETA[n], n=0..31. The redirection table has
2080          * 128-entries in 32 registers
2081          */
2082         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2083                 idx = i / RTE_RETA_GROUP_SIZE;
2084                 shift = i % RTE_RETA_GROUP_SIZE;
2085                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2086                                 BIT_MASK_PER_UINT32);
2087                 if (mask == 0)
2088                         continue;
2089
2090                 reta = 0;
2091                 if (mask != BIT_MASK_PER_UINT32)
2092                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2093
2094                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2095                         if (mask & (0x1 << j)) {
2096                                 if (mask != 0xF)
2097                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
2098                                 reta |= reta_conf[idx].reta[shift + j] <<
2099                                                 (CHAR_BIT * j);
2100                         }
2101                 }
2102                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2103         }
2104
2105         return 0;
2106 }
2107
2108 static int
2109 fm10k_reta_query(struct rte_eth_dev *dev,
2110                         struct rte_eth_rss_reta_entry64 *reta_conf,
2111                         uint16_t reta_size)
2112 {
2113         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2114         uint16_t i, j, idx, shift;
2115         uint8_t mask;
2116         uint32_t reta;
2117
2118         PMD_INIT_FUNC_TRACE();
2119
2120         if (reta_size < FM10K_MAX_RSS_INDICES) {
2121                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2122                         "(%d) doesn't match the number hardware can supported "
2123                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2124                 return -EINVAL;
2125         }
2126
2127         /*
2128          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2129          * 128-entries in 32 registers
2130          */
2131         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2132                 idx = i / RTE_RETA_GROUP_SIZE;
2133                 shift = i % RTE_RETA_GROUP_SIZE;
2134                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2135                                 BIT_MASK_PER_UINT32);
2136                 if (mask == 0)
2137                         continue;
2138
2139                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2140                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2141                         if (mask & (0x1 << j))
2142                                 reta_conf[idx].reta[shift + j] = ((reta >>
2143                                         CHAR_BIT * j) & UINT8_MAX);
2144                 }
2145         }
2146
2147         return 0;
2148 }
2149
2150 static int
2151 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2152         struct rte_eth_rss_conf *rss_conf)
2153 {
2154         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2155         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2156         uint32_t mrqc;
2157         uint64_t hf = rss_conf->rss_hf;
2158         int i;
2159
2160         PMD_INIT_FUNC_TRACE();
2161
2162         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2163                 FM10K_RSSRK_ENTRIES_PER_REG)
2164                 return -EINVAL;
2165
2166         if (hf == 0)
2167                 return -EINVAL;
2168
2169         mrqc = 0;
2170         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2171         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2172         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2173         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2174         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2175         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2176         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2177         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2178         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2179
2180         /* If the mapping doesn't fit any supported, return */
2181         if (mrqc == 0)
2182                 return -EINVAL;
2183
2184         if (key != NULL)
2185                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2186                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2187
2188         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2189
2190         return 0;
2191 }
2192
2193 static int
2194 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2195         struct rte_eth_rss_conf *rss_conf)
2196 {
2197         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2198         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2199         uint32_t mrqc;
2200         uint64_t hf;
2201         int i;
2202
2203         PMD_INIT_FUNC_TRACE();
2204
2205         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2206                                 FM10K_RSSRK_ENTRIES_PER_REG)
2207                 return -EINVAL;
2208
2209         if (key != NULL)
2210                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2211                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2212
2213         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2214         hf = 0;
2215         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2216         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2217         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2218         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2219         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2220         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2221         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2222         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2223         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2224
2225         rss_conf->rss_hf = hf;
2226
2227         return 0;
2228 }
2229
2230 static void
2231 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2232 {
2233         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2234         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2235
2236         /* Bind all local non-queue interrupt to vector 0 */
2237         int_map |= FM10K_MISC_VEC_ID;
2238
2239         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2240         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2241         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2242         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2243         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2244         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2245
2246         /* Enable misc causes */
2247         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2248                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2249                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2250                                 FM10K_EIMR_ENABLE(MAILBOX) |
2251                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2252                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2253                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2254                                 FM10K_EIMR_ENABLE(VFLR));
2255
2256         /* Enable ITR 0 */
2257         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2258                                         FM10K_ITR_MASK_CLEAR);
2259         FM10K_WRITE_FLUSH(hw);
2260 }
2261
2262 static void
2263 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2264 {
2265         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2266         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2267
2268         int_map |= FM10K_MISC_VEC_ID;
2269
2270         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2271         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2272         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2273         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2274         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2275         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2276
2277         /* Disable misc causes */
2278         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2279                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2280                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2281                                 FM10K_EIMR_DISABLE(MAILBOX) |
2282                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2283                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2284                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2285                                 FM10K_EIMR_DISABLE(VFLR));
2286
2287         /* Disable ITR 0 */
2288         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2289         FM10K_WRITE_FLUSH(hw);
2290 }
2291
2292 static void
2293 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2294 {
2295         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2296         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2297
2298         /* Bind all local non-queue interrupt to vector 0 */
2299         int_map |= FM10K_MISC_VEC_ID;
2300
2301         /* Only INT 0 available, other 15 are reserved. */
2302         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2303
2304         /* Enable ITR 0 */
2305         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2306                                         FM10K_ITR_MASK_CLEAR);
2307         FM10K_WRITE_FLUSH(hw);
2308 }
2309
2310 static void
2311 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2312 {
2313         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2314         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2315
2316         int_map |= FM10K_MISC_VEC_ID;
2317
2318         /* Only INT 0 available, other 15 are reserved. */
2319         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2320
2321         /* Disable ITR 0 */
2322         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2323         FM10K_WRITE_FLUSH(hw);
2324 }
2325
2326 static int
2327 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2328 {
2329         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2330
2331         /* Enable ITR */
2332         if (hw->mac.type == fm10k_mac_pf)
2333                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, queue_id)),
2334                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2335         else
2336                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, queue_id)),
2337                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2338         rte_intr_enable(&dev->pci_dev->intr_handle);
2339         return 0;
2340 }
2341
2342 static int
2343 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2344 {
2345         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2346
2347         /* Disable ITR */
2348         if (hw->mac.type == fm10k_mac_pf)
2349                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, queue_id)),
2350                         FM10K_ITR_MASK_SET);
2351         else
2352                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, queue_id)),
2353                         FM10K_ITR_MASK_SET);
2354         return 0;
2355 }
2356
2357 static int
2358 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2359 {
2360         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2361         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
2362         uint32_t intr_vector, vec;
2363         uint16_t queue_id;
2364         int result = 0;
2365
2366         /* fm10k needs one separate interrupt for mailbox,
2367          * so only drivers which support multiple interrupt vectors
2368          * e.g. vfio-pci can work for fm10k interrupt mode
2369          */
2370         if (!rte_intr_cap_multiple(intr_handle) ||
2371                         dev->data->dev_conf.intr_conf.rxq == 0)
2372                 return result;
2373
2374         intr_vector = dev->data->nb_rx_queues;
2375
2376         /* disable interrupt first */
2377         rte_intr_disable(&dev->pci_dev->intr_handle);
2378         if (hw->mac.type == fm10k_mac_pf)
2379                 fm10k_dev_disable_intr_pf(dev);
2380         else
2381                 fm10k_dev_disable_intr_vf(dev);
2382
2383         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2384                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2385                 result = -EIO;
2386         }
2387
2388         if (rte_intr_dp_is_en(intr_handle) && !result) {
2389                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2390                         dev->data->nb_rx_queues * sizeof(int), 0);
2391                 if (intr_handle->intr_vec) {
2392                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2393                                         queue_id < dev->data->nb_rx_queues;
2394                                         queue_id++) {
2395                                 intr_handle->intr_vec[queue_id] = vec;
2396                                 if (vec < intr_handle->nb_efd - 1
2397                                                 + FM10K_RX_VEC_START)
2398                                         vec++;
2399                         }
2400                 } else {
2401                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2402                                 " intr_vec", dev->data->nb_rx_queues);
2403                         rte_intr_efd_disable(intr_handle);
2404                         result = -ENOMEM;
2405                 }
2406         }
2407
2408         if (hw->mac.type == fm10k_mac_pf)
2409                 fm10k_dev_enable_intr_pf(dev);
2410         else
2411                 fm10k_dev_enable_intr_vf(dev);
2412         rte_intr_enable(&dev->pci_dev->intr_handle);
2413         hw->mac.ops.update_int_moderator(hw);
2414         return result;
2415 }
2416
2417 static int
2418 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2419 {
2420         struct fm10k_fault fault;
2421         int err;
2422         const char *estr = "Unknown error";
2423
2424         /* Process PCA fault */
2425         if (eicr & FM10K_EICR_PCA_FAULT) {
2426                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2427                 if (err)
2428                         goto error;
2429                 switch (fault.type) {
2430                 case PCA_NO_FAULT:
2431                         estr = "PCA_NO_FAULT"; break;
2432                 case PCA_UNMAPPED_ADDR:
2433                         estr = "PCA_UNMAPPED_ADDR"; break;
2434                 case PCA_BAD_QACCESS_PF:
2435                         estr = "PCA_BAD_QACCESS_PF"; break;
2436                 case PCA_BAD_QACCESS_VF:
2437                         estr = "PCA_BAD_QACCESS_VF"; break;
2438                 case PCA_MALICIOUS_REQ:
2439                         estr = "PCA_MALICIOUS_REQ"; break;
2440                 case PCA_POISONED_TLP:
2441                         estr = "PCA_POISONED_TLP"; break;
2442                 case PCA_TLP_ABORT:
2443                         estr = "PCA_TLP_ABORT"; break;
2444                 default:
2445                         goto error;
2446                 }
2447                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2448                         estr, fault.func ? "VF" : "PF", fault.func,
2449                         fault.address, fault.specinfo);
2450         }
2451
2452         /* Process THI fault */
2453         if (eicr & FM10K_EICR_THI_FAULT) {
2454                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2455                 if (err)
2456                         goto error;
2457                 switch (fault.type) {
2458                 case THI_NO_FAULT:
2459                         estr = "THI_NO_FAULT"; break;
2460                 case THI_MAL_DIS_Q_FAULT:
2461                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2462                 default:
2463                         goto error;
2464                 }
2465                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2466                         estr, fault.func ? "VF" : "PF", fault.func,
2467                         fault.address, fault.specinfo);
2468         }
2469
2470         /* Process FUM fault */
2471         if (eicr & FM10K_EICR_FUM_FAULT) {
2472                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2473                 if (err)
2474                         goto error;
2475                 switch (fault.type) {
2476                 case FUM_NO_FAULT:
2477                         estr = "FUM_NO_FAULT"; break;
2478                 case FUM_UNMAPPED_ADDR:
2479                         estr = "FUM_UNMAPPED_ADDR"; break;
2480                 case FUM_POISONED_TLP:
2481                         estr = "FUM_POISONED_TLP"; break;
2482                 case FUM_BAD_VF_QACCESS:
2483                         estr = "FUM_BAD_VF_QACCESS"; break;
2484                 case FUM_ADD_DECODE_ERR:
2485                         estr = "FUM_ADD_DECODE_ERR"; break;
2486                 case FUM_RO_ERROR:
2487                         estr = "FUM_RO_ERROR"; break;
2488                 case FUM_QPRC_CRC_ERROR:
2489                         estr = "FUM_QPRC_CRC_ERROR"; break;
2490                 case FUM_CSR_TIMEOUT:
2491                         estr = "FUM_CSR_TIMEOUT"; break;
2492                 case FUM_INVALID_TYPE:
2493                         estr = "FUM_INVALID_TYPE"; break;
2494                 case FUM_INVALID_LENGTH:
2495                         estr = "FUM_INVALID_LENGTH"; break;
2496                 case FUM_INVALID_BE:
2497                         estr = "FUM_INVALID_BE"; break;
2498                 case FUM_INVALID_ALIGN:
2499                         estr = "FUM_INVALID_ALIGN"; break;
2500                 default:
2501                         goto error;
2502                 }
2503                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2504                         estr, fault.func ? "VF" : "PF", fault.func,
2505                         fault.address, fault.specinfo);
2506         }
2507
2508         return 0;
2509 error:
2510         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2511         return err;
2512 }
2513
2514 /**
2515  * PF interrupt handler triggered by NIC for handling specific interrupt.
2516  *
2517  * @param handle
2518  *  Pointer to interrupt handle.
2519  * @param param
2520  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2521  *
2522  * @return
2523  *  void
2524  */
2525 static void
2526 fm10k_dev_interrupt_handler_pf(
2527                         __rte_unused struct rte_intr_handle *handle,
2528                         void *param)
2529 {
2530         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2531         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2532         uint32_t cause, status;
2533
2534         if (hw->mac.type != fm10k_mac_pf)
2535                 return;
2536
2537         cause = FM10K_READ_REG(hw, FM10K_EICR);
2538
2539         /* Handle PCI fault cases */
2540         if (cause & FM10K_EICR_FAULT_MASK) {
2541                 PMD_INIT_LOG(ERR, "INT: find fault!");
2542                 fm10k_dev_handle_fault(hw, cause);
2543         }
2544
2545         /* Handle switch up/down */
2546         if (cause & FM10K_EICR_SWITCHNOTREADY)
2547                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2548
2549         if (cause & FM10K_EICR_SWITCHREADY)
2550                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2551
2552         /* Handle mailbox message */
2553         fm10k_mbx_lock(hw);
2554         hw->mbx.ops.process(hw, &hw->mbx);
2555         fm10k_mbx_unlock(hw);
2556
2557         /* Handle SRAM error */
2558         if (cause & FM10K_EICR_SRAMERROR) {
2559                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2560
2561                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2562                 /* Write to clear pending bits */
2563                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2564
2565                 /* Todo: print out error message after shared code  updates */
2566         }
2567
2568         /* Clear these 3 events if having any */
2569         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2570                  FM10K_EICR_SWITCHREADY;
2571         if (cause)
2572                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2573
2574         /* Re-enable interrupt from device side */
2575         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2576                                         FM10K_ITR_MASK_CLEAR);
2577         /* Re-enable interrupt from host side */
2578         rte_intr_enable(&(dev->pci_dev->intr_handle));
2579 }
2580
2581 /**
2582  * VF interrupt handler triggered by NIC for handling specific interrupt.
2583  *
2584  * @param handle
2585  *  Pointer to interrupt handle.
2586  * @param param
2587  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2588  *
2589  * @return
2590  *  void
2591  */
2592 static void
2593 fm10k_dev_interrupt_handler_vf(
2594                         __rte_unused struct rte_intr_handle *handle,
2595                         void *param)
2596 {
2597         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2598         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2599
2600         if (hw->mac.type != fm10k_mac_vf)
2601                 return;
2602
2603         /* Handle mailbox message if lock is acquired */
2604         fm10k_mbx_lock(hw);
2605         hw->mbx.ops.process(hw, &hw->mbx);
2606         fm10k_mbx_unlock(hw);
2607
2608         /* Re-enable interrupt from device side */
2609         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2610                                         FM10K_ITR_MASK_CLEAR);
2611         /* Re-enable interrupt from host side */
2612         rte_intr_enable(&(dev->pci_dev->intr_handle));
2613 }
2614
2615 /* Mailbox message handler in VF */
2616 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2617         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2618         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2619         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2620         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2621 };
2622
2623 static int
2624 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2625 {
2626         int err = 0;
2627
2628         /* Initialize mailbox lock */
2629         fm10k_mbx_initlock(hw);
2630
2631         /* Replace default message handler with new ones */
2632         if (hw->mac.type == fm10k_mac_vf)
2633                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2634
2635         if (err) {
2636                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2637                                 err);
2638                 return err;
2639         }
2640         /* Connect to SM for PF device or PF for VF device */
2641         return hw->mbx.ops.connect(hw, &hw->mbx);
2642 }
2643
2644 static void
2645 fm10k_close_mbx_service(struct fm10k_hw *hw)
2646 {
2647         /* Disconnect from SM for PF device or PF for VF device */
2648         hw->mbx.ops.disconnect(hw, &hw->mbx);
2649 }
2650
2651 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2652         .dev_configure          = fm10k_dev_configure,
2653         .dev_start              = fm10k_dev_start,
2654         .dev_stop               = fm10k_dev_stop,
2655         .dev_close              = fm10k_dev_close,
2656         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2657         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2658         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2659         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2660         .stats_get              = fm10k_stats_get,
2661         .xstats_get             = fm10k_xstats_get,
2662         .xstats_get_names       = fm10k_xstats_get_names,
2663         .stats_reset            = fm10k_stats_reset,
2664         .xstats_reset           = fm10k_stats_reset,
2665         .link_update            = fm10k_link_update,
2666         .dev_infos_get          = fm10k_dev_infos_get,
2667         .dev_supported_ptypes_get = fm10k_dev_supported_ptypes_get,
2668         .vlan_filter_set        = fm10k_vlan_filter_set,
2669         .vlan_offload_set       = fm10k_vlan_offload_set,
2670         .mac_addr_add           = fm10k_macaddr_add,
2671         .mac_addr_remove        = fm10k_macaddr_remove,
2672         .rx_queue_start         = fm10k_dev_rx_queue_start,
2673         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2674         .tx_queue_start         = fm10k_dev_tx_queue_start,
2675         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2676         .rx_queue_setup         = fm10k_rx_queue_setup,
2677         .rx_queue_release       = fm10k_rx_queue_release,
2678         .tx_queue_setup         = fm10k_tx_queue_setup,
2679         .tx_queue_release       = fm10k_tx_queue_release,
2680         .rx_descriptor_done     = fm10k_dev_rx_descriptor_done,
2681         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2682         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2683         .reta_update            = fm10k_reta_update,
2684         .reta_query             = fm10k_reta_query,
2685         .rss_hash_update        = fm10k_rss_hash_update,
2686         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2687 };
2688
2689 static int ftag_check_handler(__rte_unused const char *key,
2690                 const char *value, __rte_unused void *opaque)
2691 {
2692         if (strcmp(value, "1"))
2693                 return -1;
2694
2695         return 0;
2696 }
2697
2698 static int
2699 fm10k_check_ftag(struct rte_devargs *devargs)
2700 {
2701         struct rte_kvargs *kvlist;
2702         const char *ftag_key = "enable_ftag";
2703
2704         if (devargs == NULL)
2705                 return 0;
2706
2707         kvlist = rte_kvargs_parse(devargs->args, NULL);
2708         if (kvlist == NULL)
2709                 return 0;
2710
2711         if (!rte_kvargs_count(kvlist, ftag_key)) {
2712                 rte_kvargs_free(kvlist);
2713                 return 0;
2714         }
2715         /* FTAG is enabled when there's key-value pair: enable_ftag=1 */
2716         if (rte_kvargs_process(kvlist, ftag_key,
2717                                 ftag_check_handler, NULL) < 0) {
2718                 rte_kvargs_free(kvlist);
2719                 return 0;
2720         }
2721         rte_kvargs_free(kvlist);
2722
2723         return 1;
2724 }
2725
2726 static void __attribute__((cold))
2727 fm10k_set_tx_function(struct rte_eth_dev *dev)
2728 {
2729         struct fm10k_tx_queue *txq;
2730         int i;
2731         int use_sse = 1;
2732         uint16_t tx_ftag_en = 0;
2733
2734         if (fm10k_check_ftag(dev->pci_dev->devargs))
2735                 tx_ftag_en = 1;
2736
2737         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2738                 txq = dev->data->tx_queues[i];
2739                 txq->tx_ftag_en = tx_ftag_en;
2740                 /* Check if Vector Tx is satisfied */
2741                 if (fm10k_tx_vec_condition_check(txq)) {
2742                         use_sse = 0;
2743                         break;
2744                 }
2745         }
2746
2747         if (use_sse) {
2748                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2749                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2750                         txq = dev->data->tx_queues[i];
2751                         fm10k_txq_vec_setup(txq);
2752                 }
2753                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2754         } else {
2755                 dev->tx_pkt_burst = fm10k_xmit_pkts;
2756                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2757         }
2758 }
2759
2760 static void __attribute__((cold))
2761 fm10k_set_rx_function(struct rte_eth_dev *dev)
2762 {
2763         struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev);
2764         uint16_t i, rx_using_sse;
2765         uint16_t rx_ftag_en = 0;
2766
2767         if (fm10k_check_ftag(dev->pci_dev->devargs))
2768                 rx_ftag_en = 1;
2769
2770         /* In order to allow Vector Rx there are a few configuration
2771          * conditions to be met.
2772          */
2773         if (!fm10k_rx_vec_condition_check(dev) &&
2774                         dev_info->rx_vec_allowed && !rx_ftag_en) {
2775                 if (dev->data->scattered_rx)
2776                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
2777                 else
2778                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
2779         } else if (dev->data->scattered_rx)
2780                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
2781         else
2782                 dev->rx_pkt_burst = fm10k_recv_pkts;
2783
2784         rx_using_sse =
2785                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
2786                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
2787
2788         if (rx_using_sse)
2789                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
2790         else
2791                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
2792
2793         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2794                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
2795
2796                 rxq->rx_using_sse = rx_using_sse;
2797                 rxq->rx_ftag_en = rx_ftag_en;
2798         }
2799 }
2800
2801 static void
2802 fm10k_params_init(struct rte_eth_dev *dev)
2803 {
2804         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2805         struct fm10k_dev_info *info = FM10K_DEV_PRIVATE_TO_INFO(dev);
2806
2807         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
2808          * there is no way to get link status without reading BAR4.  Until this
2809          * works, assume we have maximum bandwidth.
2810          * @todo - fix bus info
2811          */
2812         hw->bus_caps.speed = fm10k_bus_speed_8000;
2813         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
2814         hw->bus_caps.payload = fm10k_bus_payload_512;
2815         hw->bus.speed = fm10k_bus_speed_8000;
2816         hw->bus.width = fm10k_bus_width_pcie_x8;
2817         hw->bus.payload = fm10k_bus_payload_256;
2818
2819         info->rx_vec_allowed = true;
2820 }
2821
2822 static int
2823 eth_fm10k_dev_init(struct rte_eth_dev *dev)
2824 {
2825         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2826         int diag, i;
2827         struct fm10k_macvlan_filter_info *macvlan;
2828
2829         PMD_INIT_FUNC_TRACE();
2830
2831         dev->dev_ops = &fm10k_eth_dev_ops;
2832         dev->rx_pkt_burst = &fm10k_recv_pkts;
2833         dev->tx_pkt_burst = &fm10k_xmit_pkts;
2834
2835         /* only initialize in the primary process */
2836         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2837                 return 0;
2838
2839         rte_eth_copy_pci_info(dev, dev->pci_dev);
2840
2841         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
2842         memset(macvlan, 0, sizeof(*macvlan));
2843         /* Vendor and Device ID need to be set before init of shared code */
2844         memset(hw, 0, sizeof(*hw));
2845         hw->device_id = dev->pci_dev->id.device_id;
2846         hw->vendor_id = dev->pci_dev->id.vendor_id;
2847         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
2848         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
2849         hw->revision_id = 0;
2850         hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
2851         if (hw->hw_addr == NULL) {
2852                 PMD_INIT_LOG(ERR, "Bad mem resource."
2853                         " Try to blacklist unused devices.");
2854                 return -EIO;
2855         }
2856
2857         /* Store fm10k_adapter pointer */
2858         hw->back = dev->data->dev_private;
2859
2860         /* Initialize the shared code */
2861         diag = fm10k_init_shared_code(hw);
2862         if (diag != FM10K_SUCCESS) {
2863                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
2864                 return -EIO;
2865         }
2866
2867         /* Initialize parameters */
2868         fm10k_params_init(dev);
2869
2870         /* Initialize the hw */
2871         diag = fm10k_init_hw(hw);
2872         if (diag != FM10K_SUCCESS) {
2873                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
2874                 return -EIO;
2875         }
2876
2877         /* Initialize MAC address(es) */
2878         dev->data->mac_addrs = rte_zmalloc("fm10k",
2879                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
2880         if (dev->data->mac_addrs == NULL) {
2881                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
2882                 return -ENOMEM;
2883         }
2884
2885         diag = fm10k_read_mac_addr(hw);
2886
2887         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2888                         &dev->data->mac_addrs[0]);
2889
2890         if (diag != FM10K_SUCCESS ||
2891                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
2892
2893                 /* Generate a random addr */
2894                 eth_random_addr(hw->mac.addr);
2895                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
2896                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2897                 &dev->data->mac_addrs[0]);
2898         }
2899
2900         /* Reset the hw statistics */
2901         fm10k_stats_reset(dev);
2902
2903         /* Reset the hw */
2904         diag = fm10k_reset_hw(hw);
2905         if (diag != FM10K_SUCCESS) {
2906                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
2907                 return -EIO;
2908         }
2909
2910         /* Setup mailbox service */
2911         diag = fm10k_setup_mbx_service(hw);
2912         if (diag != FM10K_SUCCESS) {
2913                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
2914                 return -EIO;
2915         }
2916
2917         /*PF/VF has different interrupt handling mechanism */
2918         if (hw->mac.type == fm10k_mac_pf) {
2919                 /* register callback func to eal lib */
2920                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2921                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2922
2923                 /* enable MISC interrupt */
2924                 fm10k_dev_enable_intr_pf(dev);
2925         } else { /* VF */
2926                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2927                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2928
2929                 fm10k_dev_enable_intr_vf(dev);
2930         }
2931
2932         /* Enable intr after callback registered */
2933         rte_intr_enable(&(dev->pci_dev->intr_handle));
2934
2935         hw->mac.ops.update_int_moderator(hw);
2936
2937         /* Make sure Switch Manager is ready before going forward. */
2938         if (hw->mac.type == fm10k_mac_pf) {
2939                 int switch_ready = 0;
2940
2941                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
2942                         fm10k_mbx_lock(hw);
2943                         hw->mac.ops.get_host_state(hw, &switch_ready);
2944                         fm10k_mbx_unlock(hw);
2945                         if (switch_ready)
2946                                 break;
2947                         /* Delay some time to acquire async LPORT_MAP info. */
2948                         rte_delay_us(WAIT_SWITCH_MSG_US);
2949                 }
2950
2951                 if (switch_ready == 0) {
2952                         PMD_INIT_LOG(ERR, "switch is not ready");
2953                         return -1;
2954                 }
2955         }
2956
2957         /*
2958          * Below function will trigger operations on mailbox, acquire lock to
2959          * avoid race condition from interrupt handler. Operations on mailbox
2960          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
2961          * will handle and generate an interrupt to our side. Then,  FIFO in
2962          * mailbox will be touched.
2963          */
2964         fm10k_mbx_lock(hw);
2965         /* Enable port first */
2966         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2967                                         MAX_LPORT_NUM, 1);
2968
2969         /* Set unicast mode by default. App can change to other mode in other
2970          * API func.
2971          */
2972         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
2973                                         FM10K_XCAST_MODE_NONE);
2974
2975         fm10k_mbx_unlock(hw);
2976
2977         /* Make sure default VID is ready before going forward. */
2978         if (hw->mac.type == fm10k_mac_pf) {
2979                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
2980                         if (hw->mac.default_vid)
2981                                 break;
2982                         /* Delay some time to acquire async port VLAN info. */
2983                         rte_delay_us(WAIT_SWITCH_MSG_US);
2984                 }
2985
2986                 if (!hw->mac.default_vid) {
2987                         PMD_INIT_LOG(ERR, "default VID is not ready");
2988                         return -1;
2989                 }
2990         }
2991
2992         /* Add default mac address */
2993         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2994                 MAIN_VSI_POOL_NUMBER);
2995
2996         return 0;
2997 }
2998
2999 static int
3000 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
3001 {
3002         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3003
3004         PMD_INIT_FUNC_TRACE();
3005
3006         /* only uninitialize in the primary process */
3007         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3008                 return 0;
3009
3010         /* safe to close dev here */
3011         fm10k_dev_close(dev);
3012
3013         dev->dev_ops = NULL;
3014         dev->rx_pkt_burst = NULL;
3015         dev->tx_pkt_burst = NULL;
3016
3017         /* disable uio/vfio intr */
3018         rte_intr_disable(&(dev->pci_dev->intr_handle));
3019
3020         /*PF/VF has different interrupt handling mechanism */
3021         if (hw->mac.type == fm10k_mac_pf) {
3022                 /* disable interrupt */
3023                 fm10k_dev_disable_intr_pf(dev);
3024
3025                 /* unregister callback func to eal lib */
3026                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
3027                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3028         } else {
3029                 /* disable interrupt */
3030                 fm10k_dev_disable_intr_vf(dev);
3031
3032                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
3033                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3034         }
3035
3036         /* free mac memory */
3037         if (dev->data->mac_addrs) {
3038                 rte_free(dev->data->mac_addrs);
3039                 dev->data->mac_addrs = NULL;
3040         }
3041
3042         memset(hw, 0, sizeof(*hw));
3043
3044         return 0;
3045 }
3046
3047 /*
3048  * The set of PCI devices this driver supports. This driver will enable both PF
3049  * and SRIOV-VF devices.
3050  */
3051 static const struct rte_pci_id pci_id_fm10k_map[] = {
3052         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_PF) },
3053         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_SDI_FM10420_QDA2) },
3054         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_VF) },
3055         { .vendor_id = 0, /* sentinel */ },
3056 };
3057
3058 static struct eth_driver rte_pmd_fm10k = {
3059         .pci_drv = {
3060                 .name = "rte_pmd_fm10k",
3061                 .id_table = pci_id_fm10k_map,
3062                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
3063                         RTE_PCI_DRV_DETACHABLE,
3064         },
3065         .eth_dev_init = eth_fm10k_dev_init,
3066         .eth_dev_uninit = eth_fm10k_dev_uninit,
3067         .dev_private_size = sizeof(struct fm10k_adapter),
3068 };
3069
3070 /*
3071  * Driver initialization routine.
3072  * Invoked once at EAL init time.
3073  * Register itself as the [Poll Mode] Driver of PCI FM10K devices.
3074  */
3075 static int
3076 rte_pmd_fm10k_init(__rte_unused const char *name,
3077         __rte_unused const char *params)
3078 {
3079         PMD_INIT_FUNC_TRACE();
3080         rte_eth_driver_register(&rte_pmd_fm10k);
3081         return 0;
3082 }
3083
3084 static struct rte_driver rte_fm10k_driver = {
3085         .type = PMD_PDEV,
3086         .init = rte_pmd_fm10k_init,
3087 };
3088
3089 PMD_REGISTER_DRIVER(rte_fm10k_driver, fm10k);
3090 DRIVER_REGISTER_PCI_TABLE(fm10k, pci_id_fm10k_map);