drivers/net/fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2016 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_malloc.h>
  36 #include <rte_memzone.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_dev.h>
  39 #include <rte_spinlock.h>
  40 #include <rte_kvargs.h>
  41
  42 #include "fm10k.h"
  43 #include "base/fm10k_api.h"
  44
  45 /* Default delay to acquire mailbox lock */
  46 #define FM10K_MBXLOCK_DELAY_US 20
  47 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  48
  49 #define MAIN_VSI_POOL_NUMBER 0
  50
  51 /* Max try times to acquire switch status */
  52 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  53 /* Wait interval to get switch status */
  54 #define WAIT_SWITCH_MSG_US    100000
  55 /* A period of quiescence for switch */
  56 #define FM10K_SWITCH_QUIESCE_US 100000
  57 /* Number of chars per uint32 type */
  58 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  59 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  60
  61 /* default 1:1 map from queue ID to interrupt vector ID */
  62 #define Q2V(dev, queue_id) (dev->pci_dev->intr_handle.intr_vec[queue_id])
  63
  64 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  65 #define MAX_LPORT_NUM    128
  66 #define GLORT_FD_Q_BASE  0x40
  67 #define GLORT_PF_MASK    0xFFC0
  68 #define GLORT_FD_MASK    GLORT_PF_MASK
  69 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  70
  71 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  72 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  73 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  74 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  75 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  76 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  77 static int
  78 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  79 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  80         const u8 *mac, bool add, uint32_t pool);
  81 static void fm10k_tx_queue_release(void *queue);
  82 static void fm10k_rx_queue_release(void *queue);
  83 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  84 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  85 static int fm10k_check_ftag(struct rte_devargs *devargs);
  86 static int fm10k_link_update(struct rte_eth_dev *dev, int wait_to_complete);
  87
  88 struct fm10k_xstats_name_off {
  89         char name[RTE_ETH_XSTATS_NAME_SIZE];
  90         unsigned offset;
  91 };
  92
  93 struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  94         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  95         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  96         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  97         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  98         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  99         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
 100         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
 101         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
 102                 nodesc_drop)},
 103 };
 104
 105 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
 106                 sizeof(fm10k_hw_stats_strings[0]))
 107
 108 struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
 109         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
 110         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
 111         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
 112 };
 113
 114 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
 115                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
 116
 117 struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
 118         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 119         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 120 };
 121
 122 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 123                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 124
 125 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 126                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 127 static int
 128 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 129
 130 static void
 131 fm10k_mbx_initlock(struct fm10k_hw *hw)
 132 {
 133         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 134 }
 135
 136 static void
 137 fm10k_mbx_lock(struct fm10k_hw *hw)
 138 {
 139         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 140                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 141 }
 142
 143 static void
 144 fm10k_mbx_unlock(struct fm10k_hw *hw)
 145 {
 146         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 147 }
 148
 149 /* Stubs needed for linkage when vPMD is disabled */
 150 int __attribute__((weak))
 151 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 152 {
 153         return -1;
 154 }
 155
 156 uint16_t __attribute__((weak))
 157 fm10k_recv_pkts_vec(
 158         __rte_unused void *rx_queue,
 159         __rte_unused struct rte_mbuf **rx_pkts,
 160         __rte_unused uint16_t nb_pkts)
 161 {
 162         return 0;
 163 }
 164
 165 uint16_t __attribute__((weak))
 166 fm10k_recv_scattered_pkts_vec(
 167                 __rte_unused void *rx_queue,
 168                 __rte_unused struct rte_mbuf **rx_pkts,
 169                 __rte_unused uint16_t nb_pkts)
 170 {
 171         return 0;
 172 }
 173
 174 int __attribute__((weak))
 175 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 176
 177 {
 178         return -1;
 179 }
 180
 181 void __attribute__((weak))
 182 fm10k_rx_queue_release_mbufs_vec(
 183                 __rte_unused struct fm10k_rx_queue *rxq)
 184 {
 185         return;
 186 }
 187
 188 void __attribute__((weak))
 189 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 190 {
 191         return;
 192 }
 193
 194 int __attribute__((weak))
 195 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 196 {
 197         return -1;
 198 }
 199
 200 uint16_t __attribute__((weak))
 201 fm10k_xmit_pkts_vec(__rte_unused void *tx_queue,
 202                 __rte_unused struct rte_mbuf **tx_pkts,
 203                 __rte_unused uint16_t nb_pkts)
 204 {
 205         return 0;
 206 }
 207
 208 /*
 209  * reset queue to initial state, allocate software buffers used when starting
 210  * device.
 211  * return 0 on success
 212  * return -ENOMEM if buffers cannot be allocated
 213  * return -EINVAL if buffers do not satisfy alignment condition
 214  */
 215 static inline int
 216 rx_queue_reset(struct fm10k_rx_queue *q)
 217 {
 218         static const union fm10k_rx_desc zero = {{0} };
 219         uint64_t dma_addr;
 220         int i, diag;
 221         PMD_INIT_FUNC_TRACE();
 222
 223         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 224         if (diag != 0)
 225                 return -ENOMEM;
 226
 227         for (i = 0; i < q->nb_desc; ++i) {
 228                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 229                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 230                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 231                                                 q->nb_desc);
 232                         return -EINVAL;
 233                 }
 234                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 235                 q->hw_ring[i].q.pkt_addr = dma_addr;
 236                 q->hw_ring[i].q.hdr_addr = dma_addr;
 237         }
 238
 239         /* initialize extra software ring entries. Space for these extra
 240          * entries is always allocated.
 241          */
 242         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 243         for (i = 0; i < q->nb_fake_desc; ++i) {
 244                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 245                 q->hw_ring[q->nb_desc + i] = zero;
 246         }
 247
 248         q->next_dd = 0;
 249         q->next_alloc = 0;
 250         q->next_trigger = q->alloc_thresh - 1;
 251         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 252         q->rxrearm_start = 0;
 253         q->rxrearm_nb = 0;
 254
 255         return 0;
 256 }
 257
 258 /*
 259  * clean queue, descriptor rings, free software buffers used when stopping
 260  * device.
 261  */
 262 static inline void
 263 rx_queue_clean(struct fm10k_rx_queue *q)
 264 {
 265         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 266         uint32_t i;
 267         PMD_INIT_FUNC_TRACE();
 268
 269         /* zero descriptor rings */
 270         for (i = 0; i < q->nb_desc; ++i)
 271                 q->hw_ring[i] = zero;
 272
 273         /* zero faked descriptors */
 274         for (i = 0; i < q->nb_fake_desc; ++i)
 275                 q->hw_ring[q->nb_desc + i] = zero;
 276
 277         /* vPMD driver has a different way of releasing mbufs. */
 278         if (q->rx_using_sse) {
 279                 fm10k_rx_queue_release_mbufs_vec(q);
 280                 return;
 281         }
 282
 283         /* free software buffers */
 284         for (i = 0; i < q->nb_desc; ++i) {
 285                 if (q->sw_ring[i]) {
 286                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 287                         q->sw_ring[i] = NULL;
 288                 }
 289         }
 290 }
 291
 292 /*
 293  * free all queue memory used when releasing the queue (i.e. configure)
 294  */
 295 static inline void
 296 rx_queue_free(struct fm10k_rx_queue *q)
 297 {
 298         PMD_INIT_FUNC_TRACE();
 299         if (q) {
 300                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 301                 rx_queue_clean(q);
 302                 if (q->sw_ring) {
 303                         rte_free(q->sw_ring);
 304                         q->sw_ring = NULL;
 305                 }
 306                 rte_free(q);
 307                 q = NULL;
 308         }
 309 }
 310
 311 /*
 312  * disable RX queue, wait unitl HW finished necessary flush operation
 313  */
 314 static inline int
 315 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 316 {
 317         uint32_t reg, i;
 318
 319         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 320         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 321                         reg & ~FM10K_RXQCTL_ENABLE);
 322
 323         /* Wait 100us at most */
 324         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 325                 rte_delay_us(1);
 326                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 327                 if (!(reg & FM10K_RXQCTL_ENABLE))
 328                         break;
 329         }
 330
 331         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 332                 return -1;
 333
 334         return 0;
 335 }
 336
 337 /*
 338  * reset queue to initial state, allocate software buffers used when starting
 339  * device
 340  */
 341 static inline void
 342 tx_queue_reset(struct fm10k_tx_queue *q)
 343 {
 344         PMD_INIT_FUNC_TRACE();
 345         q->last_free = 0;
 346         q->next_free = 0;
 347         q->nb_used = 0;
 348         q->nb_free = q->nb_desc - 1;
 349         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 350         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 351 }
 352
 353 /*
 354  * clean queue, descriptor rings, free software buffers used when stopping
 355  * device
 356  */
 357 static inline void
 358 tx_queue_clean(struct fm10k_tx_queue *q)
 359 {
 360         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 361         uint32_t i;
 362         PMD_INIT_FUNC_TRACE();
 363
 364         /* zero descriptor rings */
 365         for (i = 0; i < q->nb_desc; ++i)
 366                 q->hw_ring[i] = zero;
 367
 368         /* free software buffers */
 369         for (i = 0; i < q->nb_desc; ++i) {
 370                 if (q->sw_ring[i]) {
 371                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 372                         q->sw_ring[i] = NULL;
 373                 }
 374         }
 375 }
 376
 377 /*
 378  * free all queue memory used when releasing the queue (i.e. configure)
 379  */
 380 static inline void
 381 tx_queue_free(struct fm10k_tx_queue *q)
 382 {
 383         PMD_INIT_FUNC_TRACE();
 384         if (q) {
 385                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 386                 tx_queue_clean(q);
 387                 if (q->rs_tracker.list) {
 388                         rte_free(q->rs_tracker.list);
 389                         q->rs_tracker.list = NULL;
 390                 }
 391                 if (q->sw_ring) {
 392                         rte_free(q->sw_ring);
 393                         q->sw_ring = NULL;
 394                 }
 395                 rte_free(q);
 396                 q = NULL;
 397         }
 398 }
 399
 400 /*
 401  * disable TX queue, wait unitl HW finished necessary flush operation
 402  */
 403 static inline int
 404 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 405 {
 406         uint32_t reg, i;
 407
 408         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 409         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 410                         reg & ~FM10K_TXDCTL_ENABLE);
 411
 412         /* Wait 100us at most */
 413         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 414                 rte_delay_us(1);
 415                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 416                 if (!(reg & FM10K_TXDCTL_ENABLE))
 417                         break;
 418         }
 419
 420         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 421                 return -1;
 422
 423         return 0;
 424 }
 425
 426 static int
 427 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 428 {
 429         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 430         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 431         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 432         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 433
 434         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 435
 436         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 437                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 438                 return -EINVAL;
 439         }
 440
 441         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 442                 return 0;
 443
 444         if (hw->mac.type == fm10k_mac_vf) {
 445                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 446                 return -EINVAL;
 447         }
 448
 449         /* Check VMDQ queue pool number */
 450         if (vmdq_conf->nb_queue_pools >
 451                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 452                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 453                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 454                         vmdq_conf->nb_queue_pools);
 455                 return -EINVAL;
 456         }
 457
 458         return 0;
 459 }
 460
 461 static const struct fm10k_txq_ops def_txq_ops = {
 462         .reset = tx_queue_reset,
 463 };
 464
 465 static int
 466 fm10k_dev_configure(struct rte_eth_dev *dev)
 467 {
 468         int ret;
 469
 470         PMD_INIT_FUNC_TRACE();
 471
 472         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 473                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 474         /* multipe queue mode checking */
 475         ret  = fm10k_check_mq_mode(dev);
 476         if (ret != 0) {
 477                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 478                             ret);
 479                 return ret;
 480         }
 481
 482         return 0;
 483 }
 484
 485 static void
 486 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 487 {
 488         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 489         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 490         uint32_t i;
 491
 492         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 493
 494         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 495                 if (!vmdq_conf->pool_map[i].pools)
 496                         continue;
 497                 fm10k_mbx_lock(hw);
 498                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 499                 fm10k_mbx_unlock(hw);
 500         }
 501 }
 502
 503 static void
 504 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 505 {
 506         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 507
 508         /* Add default mac address */
 509         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 510                 MAIN_VSI_POOL_NUMBER);
 511 }
 512
 513 static void
 514 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 515 {
 516         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 517         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 518         uint32_t mrqc, *key, i, reta, j;
 519         uint64_t hf;
 520
 521 #define RSS_KEY_SIZE 40
 522         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 523                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 524                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 525                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 526                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 527                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 528         };
 529
 530         if (dev->data->nb_rx_queues == 1 ||
 531             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 532             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0) {
 533                 FM10K_WRITE_REG(hw, FM10K_MRQC(0), 0);
 534                 return;
 535         }
 536
 537         /* random key is rss_intel_key (default) or user provided (rss_key) */
 538         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 539                 key = (uint32_t *)rss_intel_key;
 540         else
 541                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 542
 543         /* Now fill our hash function seeds, 4 bytes at a time */
 544         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 545                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 546
 547         /*
 548          * Fill in redirection table
 549          * The byte-swap is needed because NIC registers are in
 550          * little-endian order.
 551          */
 552         reta = 0;
 553         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 554                 if (j == dev->data->nb_rx_queues)
 555                         j = 0;
 556                 reta = (reta << CHAR_BIT) | j;
 557                 if ((i & 3) == 3)
 558                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 559                                         rte_bswap32(reta));
 560         }
 561
 562         /*
 563          * Generate RSS hash based on packet types, TCP/UDP
 564          * port numbers and/or IPv4/v6 src and dst addresses
 565          */
 566         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 567         mrqc = 0;
 568         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 569         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 570         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 571         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 572         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 573         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 574         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 575         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 576         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 577
 578         if (mrqc == 0) {
 579                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 580                         "supported", hf);
 581                 return;
 582         }
 583
 584         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 585 }
 586
 587 static void
 588 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 589 {
 590         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 591         uint32_t i;
 592
 593         for (i = 0; i < nb_lport_new; i++) {
 594                 /* Set unicast mode by default. App can change
 595                  * to other mode in other API func.
 596                  */
 597                 fm10k_mbx_lock(hw);
 598                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 599                         FM10K_XCAST_MODE_NONE);
 600                 fm10k_mbx_unlock(hw);
 601         }
 602 }
 603
 604 static void
 605 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 606 {
 607         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 608         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 609         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 610         struct fm10k_macvlan_filter_info *macvlan;
 611         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 612         uint16_t nb_lport_new;
 613
 614         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 615         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 616
 617         fm10k_dev_rss_configure(dev);
 618
 619         /* only PF supports VMDQ */
 620         if (hw->mac.type != fm10k_mac_pf)
 621                 return;
 622
 623         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 624                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 625
 626         /* no pool number change, no need to update logic port and VLAN/MAC */
 627         if (macvlan->nb_queue_pools == nb_queue_pools)
 628                 return;
 629
 630         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 631         fm10k_dev_logic_port_update(dev, nb_lport_new);
 632
 633         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 634         memset(dev->data->mac_addrs, 0,
 635                 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 636         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
 637                 &dev->data->mac_addrs[0]);
 638         memset(macvlan, 0, sizeof(*macvlan));
 639         macvlan->nb_queue_pools = nb_queue_pools;
 640
 641         if (nb_queue_pools)
 642                 fm10k_dev_vmdq_rx_configure(dev);
 643         else
 644                 fm10k_dev_pf_main_vsi_reset(dev);
 645 }
 646
 647 static int
 648 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 649 {
 650         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 651         int i, ret;
 652         struct fm10k_tx_queue *txq;
 653         uint64_t base_addr;
 654         uint32_t size;
 655
 656         /* Disable TXINT to avoid possible interrupt */
 657         for (i = 0; i < hw->mac.max_queues; i++)
 658                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 659                                 3 << FM10K_TXINT_TIMER_SHIFT);
 660
 661         /* Setup TX queue */
 662         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 663                 txq = dev->data->tx_queues[i];
 664                 base_addr = txq->hw_ring_phys_addr;
 665                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 666
 667                 /* disable queue to avoid issues while updating state */
 668                 ret = tx_queue_disable(hw, i);
 669                 if (ret) {
 670                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 671                         return -1;
 672                 }
 673                 /* Enable use of FTAG bit in TX descriptor, PFVTCTL
 674                  * register is read-only for VF.
 675                  */
 676                 if (fm10k_check_ftag(dev->pci_dev->device.devargs)) {
 677                         if (hw->mac.type == fm10k_mac_pf) {
 678                                 FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i),
 679                                                 FM10K_PFVTCTL_FTAG_DESC_ENABLE);
 680                                 PMD_INIT_LOG(DEBUG, "FTAG mode is enabled");
 681                         } else {
 682                                 PMD_INIT_LOG(ERR, "VF FTAG is not supported.");
 683                                 return -ENOTSUP;
 684                         }
 685                 }
 686
 687                 /* set location and size for descriptor ring */
 688                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 689                                 base_addr & UINT64_LOWER_32BITS_MASK);
 690                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 691                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 692                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 693
 694                 /* assign default SGLORT for each TX queue by PF */
 695                 if (hw->mac.type == fm10k_mac_pf)
 696                         FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map);
 697         }
 698
 699         /* set up vector or scalar TX function as appropriate */
 700         fm10k_set_tx_function(dev);
 701
 702         return 0;
 703 }
 704
 705 static int
 706 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 707 {
 708         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 709         struct fm10k_macvlan_filter_info *macvlan;
 710         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
 711         int i, ret;
 712         struct fm10k_rx_queue *rxq;
 713         uint64_t base_addr;
 714         uint32_t size;
 715         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 716         uint32_t logic_port = hw->mac.dglort_map;
 717         uint16_t buf_size;
 718         uint16_t queue_stride = 0;
 719
 720         /* enable RXINT for interrupt mode */
 721         i = 0;
 722         if (rte_intr_dp_is_en(intr_handle)) {
 723                 for (; i < dev->data->nb_rx_queues; i++) {
 724                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(dev, i));
 725                         if (hw->mac.type == fm10k_mac_pf)
 726                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, i)),
 727                                         FM10K_ITR_AUTOMASK |
 728                                         FM10K_ITR_MASK_CLEAR);
 729                         else
 730                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, i)),
 731                                         FM10K_ITR_AUTOMASK |
 732                                         FM10K_ITR_MASK_CLEAR);
 733                 }
 734         }
 735         /* Disable other RXINT to avoid possible interrupt */
 736         for (; i < hw->mac.max_queues; i++)
 737                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 738                         3 << FM10K_RXINT_TIMER_SHIFT);
 739
 740         /* Setup RX queues */
 741         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 742                 rxq = dev->data->rx_queues[i];
 743                 base_addr = rxq->hw_ring_phys_addr;
 744                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 745
 746                 /* disable queue to avoid issues while updating state */
 747                 ret = rx_queue_disable(hw, i);
 748                 if (ret) {
 749                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 750                         return -1;
 751                 }
 752
 753                 /* Setup the Base and Length of the Rx Descriptor Ring */
 754                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 755                                 base_addr & UINT64_LOWER_32BITS_MASK);
 756                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 757                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 758                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 759
 760                 /* Configure the Rx buffer size for one buff without split */
 761                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 762                         RTE_PKTMBUF_HEADROOM);
 763                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 764                  * reserved for this purpose, and the worst case could be 511B.
 765                  * But SRR reg assumes all buffers have the same size. In order
 766                  * to fill the gap, we'll have to consider the worst case and
 767                  * assume 512B is reserved. If we don't do so, it's possible
 768                  * for HW to overwrite data to next mbuf.
 769                  */
 770                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 771
 772                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 773                                 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
 774                                 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
 775
 776                 /* It adds dual VLAN length for supporting dual VLAN */
 777                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 778                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 779                         dev->data->dev_conf.rxmode.enable_scatter) {
 780                         uint32_t reg;
 781                         dev->data->scattered_rx = 1;
 782                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 783                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 784                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 785                 }
 786
 787                 /* Enable drop on empty, it's RO for VF */
 788                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 789                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 790
 791                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 792                 FM10K_WRITE_FLUSH(hw);
 793         }
 794
 795         /* Configure VMDQ/RSS if applicable */
 796         fm10k_dev_mq_rx_configure(dev);
 797
 798         /* Decide the best RX function */
 799         fm10k_set_rx_function(dev);
 800
 801         /* update RX_SGLORT for loopback suppress*/
 802         if (hw->mac.type != fm10k_mac_pf)
 803                 return 0;
 804         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 805         if (macvlan->nb_queue_pools)
 806                 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
 807         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 808                 if (i && queue_stride && !(i % queue_stride))
 809                         logic_port++;
 810                 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
 811         }
 812
 813         return 0;
 814 }
 815
 816 static int
 817 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 818 {
 819         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 820         int err = -1;
 821         uint32_t reg;
 822         struct fm10k_rx_queue *rxq;
 823
 824         PMD_INIT_FUNC_TRACE();
 825
 826         if (rx_queue_id < dev->data->nb_rx_queues) {
 827                 rxq = dev->data->rx_queues[rx_queue_id];
 828                 err = rx_queue_reset(rxq);
 829                 if (err == -ENOMEM) {
 830                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 831                         return err;
 832                 } else if (err == -EINVAL) {
 833                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 834                                 " %d", err);
 835                         return err;
 836                 }
 837
 838                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 839                  * Note: this must be done AFTER the queue is enabled on real
 840                  * hardware, but BEFORE the queue is enabled when using the
 841                  * emulation platform. Do it in both places for now and remove
 842                  * this comment and the following two register writes when the
 843                  * emulation platform is no longer being used.
 844                  */
 845                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 846                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 847
 848                 /* Set PF ownership flag for PF devices */
 849                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 850                 if (hw->mac.type == fm10k_mac_pf)
 851                         reg |= FM10K_RXQCTL_PF;
 852                 reg |= FM10K_RXQCTL_ENABLE;
 853                 /* enable RX queue */
 854                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 855                 FM10K_WRITE_FLUSH(hw);
 856
 857                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 858                  * Note: this must be done AFTER the queue is enabled
 859                  */
 860                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 861                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 862                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 863         }
 864
 865         return err;
 866 }
 867
 868 static int
 869 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 870 {
 871         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 872
 873         PMD_INIT_FUNC_TRACE();
 874
 875         if (rx_queue_id < dev->data->nb_rx_queues) {
 876                 /* Disable RX queue */
 877                 rx_queue_disable(hw, rx_queue_id);
 878
 879                 /* Free mbuf and clean HW ring */
 880                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 881                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 882         }
 883
 884         return 0;
 885 }
 886
 887 static int
 888 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 889 {
 890         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 891         /** @todo - this should be defined in the shared code */
 892 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 893         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 894         int err = 0;
 895
 896         PMD_INIT_FUNC_TRACE();
 897
 898         if (tx_queue_id < dev->data->nb_tx_queues) {
 899                 struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 900
 901                 q->ops->reset(q);
 902
 903                 /* reset head and tail pointers */
 904                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 905                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 906
 907                 /* enable TX queue */
 908                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 909                                         FM10K_TXDCTL_ENABLE | txdctl);
 910                 FM10K_WRITE_FLUSH(hw);
 911                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 912         } else
 913                 err = -1;
 914
 915         return err;
 916 }
 917
 918 static int
 919 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 920 {
 921         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 922
 923         PMD_INIT_FUNC_TRACE();
 924
 925         if (tx_queue_id < dev->data->nb_tx_queues) {
 926                 tx_queue_disable(hw, tx_queue_id);
 927                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 928                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 929         }
 930
 931         return 0;
 932 }
 933
 934 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 935 {
 936         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 937                 != FM10K_DGLORTMAP_NONE);
 938 }
 939
 940 static void
 941 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 942 {
 943         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 944         int status;
 945
 946         PMD_INIT_FUNC_TRACE();
 947
 948         /* Return if it didn't acquire valid glort range */
 949         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 950                 return;
 951
 952         fm10k_mbx_lock(hw);
 953         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 954                                 FM10K_XCAST_MODE_PROMISC);
 955         fm10k_mbx_unlock(hw);
 956
 957         if (status != FM10K_SUCCESS)
 958                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 959 }
 960
 961 static void
 962 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 963 {
 964         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 965         uint8_t mode;
 966         int status;
 967
 968         PMD_INIT_FUNC_TRACE();
 969
 970         /* Return if it didn't acquire valid glort range */
 971         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 972                 return;
 973
 974         if (dev->data->all_multicast == 1)
 975                 mode = FM10K_XCAST_MODE_ALLMULTI;
 976         else
 977                 mode = FM10K_XCAST_MODE_NONE;
 978
 979         fm10k_mbx_lock(hw);
 980         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 981                                 mode);
 982         fm10k_mbx_unlock(hw);
 983
 984         if (status != FM10K_SUCCESS)
 985                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 986 }
 987
 988 static void
 989 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 990 {
 991         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 992         int status;
 993
 994         PMD_INIT_FUNC_TRACE();
 995
 996         /* Return if it didn't acquire valid glort range */
 997         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 998                 return;
 999
1000         /* If promiscuous mode is enabled, it doesn't make sense to enable
1001          * allmulticast and disable promiscuous since fm10k only can select
1002          * one of the modes.
1003          */
1004         if (dev->data->promiscuous) {
1005                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
1006                         "needn't enable allmulticast");
1007                 return;
1008         }
1009
1010         fm10k_mbx_lock(hw);
1011         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1012                                 FM10K_XCAST_MODE_ALLMULTI);
1013         fm10k_mbx_unlock(hw);
1014
1015         if (status != FM10K_SUCCESS)
1016                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
1017 }
1018
1019 static void
1020 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
1021 {
1022         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1023         int status;
1024
1025         PMD_INIT_FUNC_TRACE();
1026
1027         /* Return if it didn't acquire valid glort range */
1028         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1029                 return;
1030
1031         if (dev->data->promiscuous) {
1032                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1033                         "since promisc mode is enabled");
1034                 return;
1035         }
1036
1037         fm10k_mbx_lock(hw);
1038         /* Change mode to unicast mode */
1039         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1040                                 FM10K_XCAST_MODE_NONE);
1041         fm10k_mbx_unlock(hw);
1042
1043         if (status != FM10K_SUCCESS)
1044                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1045 }
1046
1047 static void
1048 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1049 {
1050         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1051         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1052         uint16_t nb_queue_pools;
1053         struct fm10k_macvlan_filter_info *macvlan;
1054
1055         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1056         nb_queue_pools = macvlan->nb_queue_pools;
1057         pool_len = nb_queue_pools ? rte_fls_u32(nb_queue_pools - 1) : 0;
1058         rss_len = rte_fls_u32(dev->data->nb_rx_queues - 1) - pool_len;
1059
1060         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1061         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1062         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1063                         hw->mac.dglort_map;
1064         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1065         /* Configure VMDQ/RSS DGlort Decoder */
1066         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1067
1068         /* Flow Director configurations, only queue number is valid. */
1069         dglortdec = rte_fls_u32(dev->data->nb_rx_queues - 1);
1070         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1071                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1072         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1073         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1074
1075         /* Invalidate all other GLORT entries */
1076         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1077                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1078                                 FM10K_DGLORTMAP_NONE);
1079 }
1080
1081 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1082 static int
1083 fm10k_dev_start(struct rte_eth_dev *dev)
1084 {
1085         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1086         int i, diag;
1087
1088         PMD_INIT_FUNC_TRACE();
1089
1090         /* stop, init, then start the hw */
1091         diag = fm10k_stop_hw(hw);
1092         if (diag != FM10K_SUCCESS) {
1093                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1094                 return -EIO;
1095         }
1096
1097         diag = fm10k_init_hw(hw);
1098         if (diag != FM10K_SUCCESS) {
1099                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1100                 return -EIO;
1101         }
1102
1103         diag = fm10k_start_hw(hw);
1104         if (diag != FM10K_SUCCESS) {
1105                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1106                 return -EIO;
1107         }
1108
1109         diag = fm10k_dev_tx_init(dev);
1110         if (diag) {
1111                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1112                 return diag;
1113         }
1114
1115         if (fm10k_dev_rxq_interrupt_setup(dev))
1116                 return -EIO;
1117
1118         diag = fm10k_dev_rx_init(dev);
1119         if (diag) {
1120                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1121                 return diag;
1122         }
1123
1124         if (hw->mac.type == fm10k_mac_pf)
1125                 fm10k_dev_dglort_map_configure(dev);
1126
1127         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1128                 struct fm10k_rx_queue *rxq;
1129                 rxq = dev->data->rx_queues[i];
1130
1131                 if (rxq->rx_deferred_start)
1132                         continue;
1133                 diag = fm10k_dev_rx_queue_start(dev, i);
1134                 if (diag != 0) {
1135                         int j;
1136                         for (j = 0; j < i; ++j)
1137                                 rx_queue_clean(dev->data->rx_queues[j]);
1138                         return diag;
1139                 }
1140         }
1141
1142         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1143                 struct fm10k_tx_queue *txq;
1144                 txq = dev->data->tx_queues[i];
1145
1146                 if (txq->tx_deferred_start)
1147                         continue;
1148                 diag = fm10k_dev_tx_queue_start(dev, i);
1149                 if (diag != 0) {
1150                         int j;
1151                         for (j = 0; j < i; ++j)
1152                                 tx_queue_clean(dev->data->tx_queues[j]);
1153                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1154                                 rx_queue_clean(dev->data->rx_queues[j]);
1155                         return diag;
1156                 }
1157         }
1158
1159         /* Update default vlan when not in VMDQ mode */
1160         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1161                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1162
1163         fm10k_link_update(dev, 0);
1164
1165         return 0;
1166 }
1167
1168 static void
1169 fm10k_dev_stop(struct rte_eth_dev *dev)
1170 {
1171         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1172         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1173         int i;
1174
1175         PMD_INIT_FUNC_TRACE();
1176
1177         if (dev->data->tx_queues)
1178                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1179                         fm10k_dev_tx_queue_stop(dev, i);
1180
1181         if (dev->data->rx_queues)
1182                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1183                         fm10k_dev_rx_queue_stop(dev, i);
1184
1185         /* Disable datapath event */
1186         if (rte_intr_dp_is_en(intr_handle)) {
1187                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1188                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1189                                 3 << FM10K_RXINT_TIMER_SHIFT);
1190                         if (hw->mac.type == fm10k_mac_pf)
1191                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, i)),
1192                                         FM10K_ITR_MASK_SET);
1193                         else
1194                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, i)),
1195                                         FM10K_ITR_MASK_SET);
1196                 }
1197         }
1198         /* Clean datapath event and queue/vec mapping */
1199         rte_intr_efd_disable(intr_handle);
1200         rte_free(intr_handle->intr_vec);
1201         intr_handle->intr_vec = NULL;
1202 }
1203
1204 static void
1205 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1206 {
1207         int i;
1208
1209         PMD_INIT_FUNC_TRACE();
1210
1211         if (dev->data->tx_queues) {
1212                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1213                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1214
1215                         tx_queue_free(txq);
1216                 }
1217         }
1218
1219         if (dev->data->rx_queues) {
1220                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1221                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1222         }
1223 }
1224
1225 static void
1226 fm10k_dev_close(struct rte_eth_dev *dev)
1227 {
1228         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1229
1230         PMD_INIT_FUNC_TRACE();
1231
1232         fm10k_mbx_lock(hw);
1233         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1234                 MAX_LPORT_NUM, false);
1235         fm10k_mbx_unlock(hw);
1236
1237         /* allow 100ms for device to quiesce */
1238         rte_delay_us(FM10K_SWITCH_QUIESCE_US);
1239
1240         /* Stop mailbox service first */
1241         fm10k_close_mbx_service(hw);
1242         fm10k_dev_stop(dev);
1243         fm10k_dev_queue_release(dev);
1244         fm10k_stop_hw(hw);
1245 }
1246
1247 static int
1248 fm10k_link_update(struct rte_eth_dev *dev,
1249         __rte_unused int wait_to_complete)
1250 {
1251         PMD_INIT_FUNC_TRACE();
1252
1253         /* The host-interface link is always up.  The speed is ~50Gbps per Gen3
1254          * x8 PCIe interface. For now, we leave the speed undefined since there
1255          * is no 50Gbps Ethernet. */
1256         dev->data->dev_link.link_speed  = 0;
1257         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1258         dev->data->dev_link.link_status = ETH_LINK_UP;
1259
1260         return 0;
1261 }
1262
1263 static int fm10k_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1264         struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit)
1265 {
1266         unsigned i, q;
1267         unsigned count = 0;
1268
1269         if (xstats_names != NULL) {
1270                 /* Note: limit checked in rte_eth_xstats_names() */
1271
1272                 /* Global stats */
1273                 for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1274                         snprintf(xstats_names[count].name,
1275                                 sizeof(xstats_names[count].name),
1276                                 "%s", fm10k_hw_stats_strings[count].name);
1277                         count++;
1278                 }
1279
1280                 /* PF queue stats */
1281                 for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1282                         for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1283                                 snprintf(xstats_names[count].name,
1284                                         sizeof(xstats_names[count].name),
1285                                         "rx_q%u_%s", q,
1286                                         fm10k_hw_stats_rx_q_strings[i].name);
1287                                 count++;
1288                         }
1289                         for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1290                                 snprintf(xstats_names[count].name,
1291                                         sizeof(xstats_names[count].name),
1292                                         "tx_q%u_%s", q,
1293                                         fm10k_hw_stats_tx_q_strings[i].name);
1294                                 count++;
1295                         }
1296                 }
1297         }
1298         return FM10K_NB_XSTATS;
1299 }
1300
1301 static int
1302 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1303                  unsigned n)
1304 {
1305         struct fm10k_hw_stats *hw_stats =
1306                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1307         unsigned i, q, count = 0;
1308
1309         if (n < FM10K_NB_XSTATS)
1310                 return FM10K_NB_XSTATS;
1311
1312         /* Global stats */
1313         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1314                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1315                         fm10k_hw_stats_strings[count].offset);
1316                 xstats[count].id = count;
1317                 count++;
1318         }
1319
1320         /* PF queue stats */
1321         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1322                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1323                         xstats[count].value =
1324                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1325                                 fm10k_hw_stats_rx_q_strings[i].offset);
1326                         xstats[count].id = count;
1327                         count++;
1328                 }
1329                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1330                         xstats[count].value =
1331                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1332                                 fm10k_hw_stats_tx_q_strings[i].offset);
1333                         xstats[count].id = count;
1334                         count++;
1335                 }
1336         }
1337
1338         return FM10K_NB_XSTATS;
1339 }
1340
1341 static void
1342 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1343 {
1344         uint64_t ipackets, opackets, ibytes, obytes;
1345         struct fm10k_hw *hw =
1346                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1347         struct fm10k_hw_stats *hw_stats =
1348                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1349         int i;
1350
1351         PMD_INIT_FUNC_TRACE();
1352
1353         fm10k_update_hw_stats(hw, hw_stats);
1354
1355         ipackets = opackets = ibytes = obytes = 0;
1356         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1357                 (i < hw->mac.max_queues); ++i) {
1358                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1359                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1360                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1361                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1362                 ipackets += stats->q_ipackets[i];
1363                 opackets += stats->q_opackets[i];
1364                 ibytes   += stats->q_ibytes[i];
1365                 obytes   += stats->q_obytes[i];
1366         }
1367         stats->ipackets = ipackets;
1368         stats->opackets = opackets;
1369         stats->ibytes = ibytes;
1370         stats->obytes = obytes;
1371 }
1372
1373 static void
1374 fm10k_stats_reset(struct rte_eth_dev *dev)
1375 {
1376         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1377         struct fm10k_hw_stats *hw_stats =
1378                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1379
1380         PMD_INIT_FUNC_TRACE();
1381
1382         memset(hw_stats, 0, sizeof(*hw_stats));
1383         fm10k_rebind_hw_stats(hw, hw_stats);
1384 }
1385
1386 static void
1387 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1388         struct rte_eth_dev_info *dev_info)
1389 {
1390         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1391
1392         PMD_INIT_FUNC_TRACE();
1393
1394         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1395         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1396         dev_info->max_rx_queues      = hw->mac.max_queues;
1397         dev_info->max_tx_queues      = hw->mac.max_queues;
1398         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1399         dev_info->max_hash_mac_addrs = 0;
1400         dev_info->max_vfs            = dev->pci_dev->max_vfs;
1401         dev_info->vmdq_pool_base     = 0;
1402         dev_info->vmdq_queue_base    = 0;
1403         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1404         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1405         dev_info->rx_offload_capa =
1406                 DEV_RX_OFFLOAD_VLAN_STRIP |
1407                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1408                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1409                 DEV_RX_OFFLOAD_TCP_CKSUM;
1410         dev_info->tx_offload_capa =
1411                 DEV_TX_OFFLOAD_VLAN_INSERT |
1412                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1413                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1414                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1415                 DEV_TX_OFFLOAD_TCP_TSO;
1416
1417         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1418         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1419
1420         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1421                 .rx_thresh = {
1422                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1423                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1424                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1425                 },
1426                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1427                 .rx_drop_en = 0,
1428         };
1429
1430         dev_info->default_txconf = (struct rte_eth_txconf) {
1431                 .tx_thresh = {
1432                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1433                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1434                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1435                 },
1436                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1437                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1438                 .txq_flags = FM10K_SIMPLE_TX_FLAG,
1439         };
1440
1441         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1442                 .nb_max = FM10K_MAX_RX_DESC,
1443                 .nb_min = FM10K_MIN_RX_DESC,
1444                 .nb_align = FM10K_MULT_RX_DESC,
1445         };
1446
1447         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1448                 .nb_max = FM10K_MAX_TX_DESC,
1449                 .nb_min = FM10K_MIN_TX_DESC,
1450                 .nb_align = FM10K_MULT_TX_DESC,
1451         };
1452
1453         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
1454                         ETH_LINK_SPEED_10G | ETH_LINK_SPEED_25G |
1455                         ETH_LINK_SPEED_40G | ETH_LINK_SPEED_100G;
1456 }
1457
1458 #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
1459 static const uint32_t *
1460 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1461 {
1462         if (dev->rx_pkt_burst == fm10k_recv_pkts ||
1463             dev->rx_pkt_burst == fm10k_recv_scattered_pkts) {
1464                 static uint32_t ptypes[] = {
1465                         /* refers to rx_desc_to_ol_flags() */
1466                         RTE_PTYPE_L2_ETHER,
1467                         RTE_PTYPE_L3_IPV4,
1468                         RTE_PTYPE_L3_IPV4_EXT,
1469                         RTE_PTYPE_L3_IPV6,
1470                         RTE_PTYPE_L3_IPV6_EXT,
1471                         RTE_PTYPE_L4_TCP,
1472                         RTE_PTYPE_L4_UDP,
1473                         RTE_PTYPE_UNKNOWN
1474                 };
1475
1476                 return ptypes;
1477         } else if (dev->rx_pkt_burst == fm10k_recv_pkts_vec ||
1478                    dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec) {
1479                 static uint32_t ptypes_vec[] = {
1480                         /* refers to fm10k_desc_to_pktype_v() */
1481                         RTE_PTYPE_L3_IPV4,
1482                         RTE_PTYPE_L3_IPV4_EXT,
1483                         RTE_PTYPE_L3_IPV6,
1484                         RTE_PTYPE_L3_IPV6_EXT,
1485                         RTE_PTYPE_L4_TCP,
1486                         RTE_PTYPE_L4_UDP,
1487                         RTE_PTYPE_TUNNEL_GENEVE,
1488                         RTE_PTYPE_TUNNEL_NVGRE,
1489                         RTE_PTYPE_TUNNEL_VXLAN,
1490                         RTE_PTYPE_TUNNEL_GRE,
1491                         RTE_PTYPE_UNKNOWN
1492                 };
1493
1494                 return ptypes_vec;
1495         }
1496
1497         return NULL;
1498 }
1499 #else
1500 static const uint32_t *
1501 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1502 {
1503         return NULL;
1504 }
1505 #endif
1506
1507 static int
1508 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1509 {
1510         s32 result;
1511         uint16_t mac_num = 0;
1512         uint32_t vid_idx, vid_bit, mac_index;
1513         struct fm10k_hw *hw;
1514         struct fm10k_macvlan_filter_info *macvlan;
1515         struct rte_eth_dev_data *data = dev->data;
1516
1517         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1518         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1519
1520         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1521                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1522                 return -EINVAL;
1523         }
1524
1525         if (vlan_id > ETH_VLAN_ID_MAX) {
1526                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1527                 return -EINVAL;
1528         }
1529
1530         vid_idx = FM10K_VFTA_IDX(vlan_id);
1531         vid_bit = FM10K_VFTA_BIT(vlan_id);
1532         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1533         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1534                 return 0;
1535         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1536         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1537                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1538                         "in the VLAN filter table");
1539                 return -EINVAL;
1540         }
1541
1542         fm10k_mbx_lock(hw);
1543         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1544         fm10k_mbx_unlock(hw);
1545         if (result != FM10K_SUCCESS) {
1546                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1547                 return -EIO;
1548         }
1549
1550         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1551                         (result == FM10K_SUCCESS); mac_index++) {
1552                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1553                         continue;
1554                 if (mac_num > macvlan->mac_num - 1) {
1555                         PMD_INIT_LOG(ERR, "MAC address number "
1556                                         "not match");
1557                         break;
1558                 }
1559                 fm10k_mbx_lock(hw);
1560                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1561                         data->mac_addrs[mac_index].addr_bytes,
1562                         vlan_id, on, 0);
1563                 fm10k_mbx_unlock(hw);
1564                 mac_num++;
1565         }
1566         if (result != FM10K_SUCCESS) {
1567                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1568                 return -EIO;
1569         }
1570
1571         if (on) {
1572                 macvlan->vlan_num++;
1573                 macvlan->vfta[vid_idx] |= vid_bit;
1574         } else {
1575                 macvlan->vlan_num--;
1576                 macvlan->vfta[vid_idx] &= ~vid_bit;
1577         }
1578         return 0;
1579 }
1580
1581 static void
1582 fm10k_vlan_offload_set(__rte_unused struct rte_eth_dev *dev, int mask)
1583 {
1584         if (mask & ETH_VLAN_STRIP_MASK) {
1585                 if (!dev->data->dev_conf.rxmode.hw_vlan_strip)
1586                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1587                                         "always on in fm10k");
1588         }
1589
1590         if (mask & ETH_VLAN_EXTEND_MASK) {
1591                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1592                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1593                                         "supported in fm10k");
1594         }
1595
1596         if (mask & ETH_VLAN_FILTER_MASK) {
1597                 if (!dev->data->dev_conf.rxmode.hw_vlan_filter)
1598                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1599         }
1600 }
1601
1602 /* Add/Remove a MAC address, and update filters to main VSI */
1603 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1604                 const u8 *mac, bool add, uint32_t pool)
1605 {
1606         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1607         struct fm10k_macvlan_filter_info *macvlan;
1608         uint32_t i, j, k;
1609
1610         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1611
1612         if (pool != MAIN_VSI_POOL_NUMBER) {
1613                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1614                         "mac to pool %u", pool);
1615                 return;
1616         }
1617         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1618                 if (!macvlan->vfta[j])
1619                         continue;
1620                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1621                         if (!(macvlan->vfta[j] & (1 << k)))
1622                                 continue;
1623                         if (i + 1 > macvlan->vlan_num) {
1624                                 PMD_INIT_LOG(ERR, "vlan number not match");
1625                                 return;
1626                         }
1627                         fm10k_mbx_lock(hw);
1628                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1629                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1630                         fm10k_mbx_unlock(hw);
1631                         i++;
1632                 }
1633         }
1634 }
1635
1636 /* Add/Remove a MAC address, and update filters to VMDQ */
1637 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1638                 const u8 *mac, bool add, uint32_t pool)
1639 {
1640         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1641         struct fm10k_macvlan_filter_info *macvlan;
1642         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1643         uint32_t i;
1644
1645         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1646         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1647
1648         if (pool > macvlan->nb_queue_pools) {
1649                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1650                         " Max pool is %u",
1651                         pool, macvlan->nb_queue_pools);
1652                 return;
1653         }
1654         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1655                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1656                         continue;
1657                 fm10k_mbx_lock(hw);
1658                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1659                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1660                 fm10k_mbx_unlock(hw);
1661         }
1662 }
1663
1664 /* Add/Remove a MAC address, and update filters */
1665 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1666                 const u8 *mac, bool add, uint32_t pool)
1667 {
1668         struct fm10k_macvlan_filter_info *macvlan;
1669
1670         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1671
1672         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1673                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1674         else
1675                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1676
1677         if (add)
1678                 macvlan->mac_num++;
1679         else
1680                 macvlan->mac_num--;
1681 }
1682
1683 /* Add a MAC address, and update filters */
1684 static void
1685 fm10k_macaddr_add(struct rte_eth_dev *dev,
1686                 struct ether_addr *mac_addr,
1687                 uint32_t index,
1688                 uint32_t pool)
1689 {
1690         struct fm10k_macvlan_filter_info *macvlan;
1691
1692         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1693         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1694         macvlan->mac_vmdq_id[index] = pool;
1695 }
1696
1697 /* Remove a MAC address, and update filters */
1698 static void
1699 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1700 {
1701         struct rte_eth_dev_data *data = dev->data;
1702         struct fm10k_macvlan_filter_info *macvlan;
1703
1704         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1705         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1706                         FALSE, macvlan->mac_vmdq_id[index]);
1707         macvlan->mac_vmdq_id[index] = 0;
1708 }
1709
1710 static inline int
1711 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1712 {
1713         if ((request < min) || (request > max) || ((request % mult) != 0))
1714                 return -1;
1715         else
1716                 return 0;
1717 }
1718
1719
1720 static inline int
1721 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1722 {
1723         if ((request < min) || (request > max) || ((div % request) != 0))
1724                 return -1;
1725         else
1726                 return 0;
1727 }
1728
1729 static inline int
1730 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1731 {
1732         uint16_t rx_free_thresh;
1733
1734         if (conf->rx_free_thresh == 0)
1735                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1736         else
1737                 rx_free_thresh = conf->rx_free_thresh;
1738
1739         /* make sure the requested threshold satisfies the constraints */
1740         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1741                         FM10K_RX_FREE_THRESH_MAX(q),
1742                         FM10K_RX_FREE_THRESH_DIV(q),
1743                         rx_free_thresh)) {
1744                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1745                         "less than or equal to %u, "
1746                         "greater than or equal to %u, "
1747                         "and a divisor of %u",
1748                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1749                         FM10K_RX_FREE_THRESH_MIN(q),
1750                         FM10K_RX_FREE_THRESH_DIV(q));
1751                 return -EINVAL;
1752         }
1753
1754         q->alloc_thresh = rx_free_thresh;
1755         q->drop_en = conf->rx_drop_en;
1756         q->rx_deferred_start = conf->rx_deferred_start;
1757
1758         return 0;
1759 }
1760
1761 /*
1762  * Hardware requires specific alignment for Rx packet buffers. At
1763  * least one of the following two conditions must be satisfied.
1764  *  1. Address is 512B aligned
1765  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1766  *
1767  * As such, the driver may need to adjust the DMA address within the
1768  * buffer by up to 512B.
1769  *
1770  * return 1 if the element size is valid, otherwise return 0.
1771  */
1772 static int
1773 mempool_element_size_valid(struct rte_mempool *mp)
1774 {
1775         uint32_t min_size;
1776
1777         /* elt_size includes mbuf header and headroom */
1778         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1779                         RTE_PKTMBUF_HEADROOM;
1780
1781         /* account for up to 512B of alignment */
1782         min_size -= FM10K_RX_DATABUF_ALIGN;
1783
1784         /* sanity check for overflow */
1785         if (min_size > mp->elt_size)
1786                 return 0;
1787
1788         /* size is valid */
1789         return 1;
1790 }
1791
1792 static int
1793 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1794         uint16_t nb_desc, unsigned int socket_id,
1795         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1796 {
1797         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1798         struct fm10k_dev_info *dev_info =
1799                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1800         struct fm10k_rx_queue *q;
1801         const struct rte_memzone *mz;
1802
1803         PMD_INIT_FUNC_TRACE();
1804
1805         /* make sure the mempool element size can account for alignment. */
1806         if (!mempool_element_size_valid(mp)) {
1807                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1808                 return -EINVAL;
1809         }
1810
1811         /* make sure a valid number of descriptors have been requested */
1812         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1813                                 FM10K_MULT_RX_DESC, nb_desc)) {
1814                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1815                         "less than or equal to %"PRIu32", "
1816                         "greater than or equal to %u, "
1817                         "and a multiple of %u",
1818                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1819                         FM10K_MULT_RX_DESC);
1820                 return -EINVAL;
1821         }
1822
1823         /*
1824          * if this queue existed already, free the associated memory. The
1825          * queue cannot be reused in case we need to allocate memory on
1826          * different socket than was previously used.
1827          */
1828         if (dev->data->rx_queues[queue_id] != NULL) {
1829                 rx_queue_free(dev->data->rx_queues[queue_id]);
1830                 dev->data->rx_queues[queue_id] = NULL;
1831         }
1832
1833         /* allocate memory for the queue structure */
1834         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1835                                 socket_id);
1836         if (q == NULL) {
1837                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1838                 return -ENOMEM;
1839         }
1840
1841         /* setup queue */
1842         q->mp = mp;
1843         q->nb_desc = nb_desc;
1844         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1845         q->port_id = dev->data->port_id;
1846         q->queue_id = queue_id;
1847         q->tail_ptr = (volatile uint32_t *)
1848                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1849         if (handle_rxconf(q, conf))
1850                 return -EINVAL;
1851
1852         /* allocate memory for the software ring */
1853         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1854                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1855                         RTE_CACHE_LINE_SIZE, socket_id);
1856         if (q->sw_ring == NULL) {
1857                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1858                 rte_free(q);
1859                 return -ENOMEM;
1860         }
1861
1862         /*
1863          * allocate memory for the hardware descriptor ring. A memzone large
1864          * enough to hold the maximum ring size is requested to allow for
1865          * resizing in later calls to the queue setup function.
1866          */
1867         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1868                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1869                                       socket_id);
1870         if (mz == NULL) {
1871                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1872                 rte_free(q->sw_ring);
1873                 rte_free(q);
1874                 return -ENOMEM;
1875         }
1876         q->hw_ring = mz->addr;
1877         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1878
1879         /* Check if number of descs satisfied Vector requirement */
1880         if (!rte_is_power_of_2(nb_desc)) {
1881                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1882                                     "preconditions - canceling the feature for "
1883                                     "the whole port[%d]",
1884                              q->queue_id, q->port_id);
1885                 dev_info->rx_vec_allowed = false;
1886         } else
1887                 fm10k_rxq_vec_setup(q);
1888
1889         dev->data->rx_queues[queue_id] = q;
1890         return 0;
1891 }
1892
1893 static void
1894 fm10k_rx_queue_release(void *queue)
1895 {
1896         PMD_INIT_FUNC_TRACE();
1897
1898         rx_queue_free(queue);
1899 }
1900
1901 static inline int
1902 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1903 {
1904         uint16_t tx_free_thresh;
1905         uint16_t tx_rs_thresh;
1906
1907         /* constraint MACROs require that tx_free_thresh is configured
1908          * before tx_rs_thresh */
1909         if (conf->tx_free_thresh == 0)
1910                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1911         else
1912                 tx_free_thresh = conf->tx_free_thresh;
1913
1914         /* make sure the requested threshold satisfies the constraints */
1915         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1916                         FM10K_TX_FREE_THRESH_MAX(q),
1917                         FM10K_TX_FREE_THRESH_DIV(q),
1918                         tx_free_thresh)) {
1919                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1920                         "less than or equal to %u, "
1921                         "greater than or equal to %u, "
1922                         "and a divisor of %u",
1923                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1924                         FM10K_TX_FREE_THRESH_MIN(q),
1925                         FM10K_TX_FREE_THRESH_DIV(q));
1926                 return -EINVAL;
1927         }
1928
1929         q->free_thresh = tx_free_thresh;
1930
1931         if (conf->tx_rs_thresh == 0)
1932                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1933         else
1934                 tx_rs_thresh = conf->tx_rs_thresh;
1935
1936         q->tx_deferred_start = conf->tx_deferred_start;
1937
1938         /* make sure the requested threshold satisfies the constraints */
1939         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1940                         FM10K_TX_RS_THRESH_MAX(q),
1941                         FM10K_TX_RS_THRESH_DIV(q),
1942                         tx_rs_thresh)) {
1943                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1944                         "less than or equal to %u, "
1945                         "greater than or equal to %u, "
1946                         "and a divisor of %u",
1947                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1948                         FM10K_TX_RS_THRESH_MIN(q),
1949                         FM10K_TX_RS_THRESH_DIV(q));
1950                 return -EINVAL;
1951         }
1952
1953         q->rs_thresh = tx_rs_thresh;
1954
1955         return 0;
1956 }
1957
1958 static int
1959 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1960         uint16_t nb_desc, unsigned int socket_id,
1961         const struct rte_eth_txconf *conf)
1962 {
1963         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1964         struct fm10k_tx_queue *q;
1965         const struct rte_memzone *mz;
1966
1967         PMD_INIT_FUNC_TRACE();
1968
1969         /* make sure a valid number of descriptors have been requested */
1970         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1971                                 FM10K_MULT_TX_DESC, nb_desc)) {
1972                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1973                         "less than or equal to %"PRIu32", "
1974                         "greater than or equal to %u, "
1975                         "and a multiple of %u",
1976                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1977                         FM10K_MULT_TX_DESC);
1978                 return -EINVAL;
1979         }
1980
1981         /*
1982          * if this queue existed already, free the associated memory. The
1983          * queue cannot be reused in case we need to allocate memory on
1984          * different socket than was previously used.
1985          */
1986         if (dev->data->tx_queues[queue_id] != NULL) {
1987                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
1988
1989                 tx_queue_free(txq);
1990                 dev->data->tx_queues[queue_id] = NULL;
1991         }
1992
1993         /* allocate memory for the queue structure */
1994         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1995                                 socket_id);
1996         if (q == NULL) {
1997                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1998                 return -ENOMEM;
1999         }
2000
2001         /* setup queue */
2002         q->nb_desc = nb_desc;
2003         q->port_id = dev->data->port_id;
2004         q->queue_id = queue_id;
2005         q->txq_flags = conf->txq_flags;
2006         q->ops = &def_txq_ops;
2007         q->tail_ptr = (volatile uint32_t *)
2008                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
2009         if (handle_txconf(q, conf))
2010                 return -EINVAL;
2011
2012         /* allocate memory for the software ring */
2013         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
2014                                         nb_desc * sizeof(struct rte_mbuf *),
2015                                         RTE_CACHE_LINE_SIZE, socket_id);
2016         if (q->sw_ring == NULL) {
2017                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
2018                 rte_free(q);
2019                 return -ENOMEM;
2020         }
2021
2022         /*
2023          * allocate memory for the hardware descriptor ring. A memzone large
2024          * enough to hold the maximum ring size is requested to allow for
2025          * resizing in later calls to the queue setup function.
2026          */
2027         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
2028                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
2029                                       socket_id);
2030         if (mz == NULL) {
2031                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
2032                 rte_free(q->sw_ring);
2033                 rte_free(q);
2034                 return -ENOMEM;
2035         }
2036         q->hw_ring = mz->addr;
2037         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
2038
2039         /*
2040          * allocate memory for the RS bit tracker. Enough slots to hold the
2041          * descriptor index for each RS bit needing to be set are required.
2042          */
2043         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
2044                                 ((nb_desc + 1) / q->rs_thresh) *
2045                                 sizeof(uint16_t),
2046                                 RTE_CACHE_LINE_SIZE, socket_id);
2047         if (q->rs_tracker.list == NULL) {
2048                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
2049                 rte_free(q->sw_ring);
2050                 rte_free(q);
2051                 return -ENOMEM;
2052         }
2053
2054         dev->data->tx_queues[queue_id] = q;
2055         return 0;
2056 }
2057
2058 static void
2059 fm10k_tx_queue_release(void *queue)
2060 {
2061         struct fm10k_tx_queue *q = queue;
2062         PMD_INIT_FUNC_TRACE();
2063
2064         tx_queue_free(q);
2065 }
2066
2067 static int
2068 fm10k_reta_update(struct rte_eth_dev *dev,
2069                         struct rte_eth_rss_reta_entry64 *reta_conf,
2070                         uint16_t reta_size)
2071 {
2072         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2073         uint16_t i, j, idx, shift;
2074         uint8_t mask;
2075         uint32_t reta;
2076
2077         PMD_INIT_FUNC_TRACE();
2078
2079         if (reta_size > FM10K_MAX_RSS_INDICES) {
2080                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2081                         "(%d) doesn't match the number hardware can supported "
2082                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2083                 return -EINVAL;
2084         }
2085
2086         /*
2087          * Update Redirection Table RETA[n], n=0..31. The redirection table has
2088          * 128-entries in 32 registers
2089          */
2090         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2091                 idx = i / RTE_RETA_GROUP_SIZE;
2092                 shift = i % RTE_RETA_GROUP_SIZE;
2093                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2094                                 BIT_MASK_PER_UINT32);
2095                 if (mask == 0)
2096                         continue;
2097
2098                 reta = 0;
2099                 if (mask != BIT_MASK_PER_UINT32)
2100                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2101
2102                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2103                         if (mask & (0x1 << j)) {
2104                                 if (mask != 0xF)
2105                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
2106                                 reta |= reta_conf[idx].reta[shift + j] <<
2107                                                 (CHAR_BIT * j);
2108                         }
2109                 }
2110                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2111         }
2112
2113         return 0;
2114 }
2115
2116 static int
2117 fm10k_reta_query(struct rte_eth_dev *dev,
2118                         struct rte_eth_rss_reta_entry64 *reta_conf,
2119                         uint16_t reta_size)
2120 {
2121         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2122         uint16_t i, j, idx, shift;
2123         uint8_t mask;
2124         uint32_t reta;
2125
2126         PMD_INIT_FUNC_TRACE();
2127
2128         if (reta_size < FM10K_MAX_RSS_INDICES) {
2129                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2130                         "(%d) doesn't match the number hardware can supported "
2131                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2132                 return -EINVAL;
2133         }
2134
2135         /*
2136          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2137          * 128-entries in 32 registers
2138          */
2139         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2140                 idx = i / RTE_RETA_GROUP_SIZE;
2141                 shift = i % RTE_RETA_GROUP_SIZE;
2142                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2143                                 BIT_MASK_PER_UINT32);
2144                 if (mask == 0)
2145                         continue;
2146
2147                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2148                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2149                         if (mask & (0x1 << j))
2150                                 reta_conf[idx].reta[shift + j] = ((reta >>
2151                                         CHAR_BIT * j) & UINT8_MAX);
2152                 }
2153         }
2154
2155         return 0;
2156 }
2157
2158 static int
2159 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2160         struct rte_eth_rss_conf *rss_conf)
2161 {
2162         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2163         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2164         uint32_t mrqc;
2165         uint64_t hf = rss_conf->rss_hf;
2166         int i;
2167
2168         PMD_INIT_FUNC_TRACE();
2169
2170         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2171                                 FM10K_RSSRK_ENTRIES_PER_REG))
2172                 return -EINVAL;
2173
2174         if (hf == 0)
2175                 return -EINVAL;
2176
2177         mrqc = 0;
2178         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2179         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2180         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2181         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2182         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2183         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2184         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2185         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2186         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2187
2188         /* If the mapping doesn't fit any supported, return */
2189         if (mrqc == 0)
2190                 return -EINVAL;
2191
2192         if (key != NULL)
2193                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2194                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2195
2196         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2197
2198         return 0;
2199 }
2200
2201 static int
2202 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2203         struct rte_eth_rss_conf *rss_conf)
2204 {
2205         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2206         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2207         uint32_t mrqc;
2208         uint64_t hf;
2209         int i;
2210
2211         PMD_INIT_FUNC_TRACE();
2212
2213         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2214                                 FM10K_RSSRK_ENTRIES_PER_REG))
2215                 return -EINVAL;
2216
2217         if (key != NULL)
2218                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2219                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2220
2221         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2222         hf = 0;
2223         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2224         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2225         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2226         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2227         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2228         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2229         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2230         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2231         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2232
2233         rss_conf->rss_hf = hf;
2234
2235         return 0;
2236 }
2237
2238 static void
2239 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2240 {
2241         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2242         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2243
2244         /* Bind all local non-queue interrupt to vector 0 */
2245         int_map |= FM10K_MISC_VEC_ID;
2246
2247         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2248         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2249         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2250         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2251         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2252         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2253
2254         /* Enable misc causes */
2255         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2256                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2257                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2258                                 FM10K_EIMR_ENABLE(MAILBOX) |
2259                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2260                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2261                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2262                                 FM10K_EIMR_ENABLE(VFLR));
2263
2264         /* Enable ITR 0 */
2265         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2266                                         FM10K_ITR_MASK_CLEAR);
2267         FM10K_WRITE_FLUSH(hw);
2268 }
2269
2270 static void
2271 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2272 {
2273         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2274         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2275
2276         int_map |= FM10K_MISC_VEC_ID;
2277
2278         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2279         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2280         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2281         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2282         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2283         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2284
2285         /* Disable misc causes */
2286         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2287                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2288                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2289                                 FM10K_EIMR_DISABLE(MAILBOX) |
2290                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2291                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2292                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2293                                 FM10K_EIMR_DISABLE(VFLR));
2294
2295         /* Disable ITR 0 */
2296         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2297         FM10K_WRITE_FLUSH(hw);
2298 }
2299
2300 static void
2301 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2302 {
2303         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2304         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2305
2306         /* Bind all local non-queue interrupt to vector 0 */
2307         int_map |= FM10K_MISC_VEC_ID;
2308
2309         /* Only INT 0 available, other 15 are reserved. */
2310         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2311
2312         /* Enable ITR 0 */
2313         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2314                                         FM10K_ITR_MASK_CLEAR);
2315         FM10K_WRITE_FLUSH(hw);
2316 }
2317
2318 static void
2319 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2320 {
2321         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2322         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2323
2324         int_map |= FM10K_MISC_VEC_ID;
2325
2326         /* Only INT 0 available, other 15 are reserved. */
2327         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2328
2329         /* Disable ITR 0 */
2330         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2331         FM10K_WRITE_FLUSH(hw);
2332 }
2333
2334 static int
2335 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2336 {
2337         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2338
2339         /* Enable ITR */
2340         if (hw->mac.type == fm10k_mac_pf)
2341                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, queue_id)),
2342                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2343         else
2344                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, queue_id)),
2345                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2346         rte_intr_enable(&dev->pci_dev->intr_handle);
2347         return 0;
2348 }
2349
2350 static int
2351 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2352 {
2353         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2354
2355         /* Disable ITR */
2356         if (hw->mac.type == fm10k_mac_pf)
2357                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, queue_id)),
2358                         FM10K_ITR_MASK_SET);
2359         else
2360                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, queue_id)),
2361                         FM10K_ITR_MASK_SET);
2362         return 0;
2363 }
2364
2365 static int
2366 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2367 {
2368         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2369         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
2370         uint32_t intr_vector, vec;
2371         uint16_t queue_id;
2372         int result = 0;
2373
2374         /* fm10k needs one separate interrupt for mailbox,
2375          * so only drivers which support multiple interrupt vectors
2376          * e.g. vfio-pci can work for fm10k interrupt mode
2377          */
2378         if (!rte_intr_cap_multiple(intr_handle) ||
2379                         dev->data->dev_conf.intr_conf.rxq == 0)
2380                 return result;
2381
2382         intr_vector = dev->data->nb_rx_queues;
2383
2384         /* disable interrupt first */
2385         rte_intr_disable(&dev->pci_dev->intr_handle);
2386         if (hw->mac.type == fm10k_mac_pf)
2387                 fm10k_dev_disable_intr_pf(dev);
2388         else
2389                 fm10k_dev_disable_intr_vf(dev);
2390
2391         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2392                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2393                 result = -EIO;
2394         }
2395
2396         if (rte_intr_dp_is_en(intr_handle) && !result) {
2397                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2398                         dev->data->nb_rx_queues * sizeof(int), 0);
2399                 if (intr_handle->intr_vec) {
2400                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2401                                         queue_id < dev->data->nb_rx_queues;
2402                                         queue_id++) {
2403                                 intr_handle->intr_vec[queue_id] = vec;
2404                                 if (vec < intr_handle->nb_efd - 1
2405                                                 + FM10K_RX_VEC_START)
2406                                         vec++;
2407                         }
2408                 } else {
2409                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2410                                 " intr_vec", dev->data->nb_rx_queues);
2411                         rte_intr_efd_disable(intr_handle);
2412                         result = -ENOMEM;
2413                 }
2414         }
2415
2416         if (hw->mac.type == fm10k_mac_pf)
2417                 fm10k_dev_enable_intr_pf(dev);
2418         else
2419                 fm10k_dev_enable_intr_vf(dev);
2420         rte_intr_enable(&dev->pci_dev->intr_handle);
2421         hw->mac.ops.update_int_moderator(hw);
2422         return result;
2423 }
2424
2425 static int
2426 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2427 {
2428         struct fm10k_fault fault;
2429         int err;
2430         const char *estr = "Unknown error";
2431
2432         /* Process PCA fault */
2433         if (eicr & FM10K_EICR_PCA_FAULT) {
2434                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2435                 if (err)
2436                         goto error;
2437                 switch (fault.type) {
2438                 case PCA_NO_FAULT:
2439                         estr = "PCA_NO_FAULT"; break;
2440                 case PCA_UNMAPPED_ADDR:
2441                         estr = "PCA_UNMAPPED_ADDR"; break;
2442                 case PCA_BAD_QACCESS_PF:
2443                         estr = "PCA_BAD_QACCESS_PF"; break;
2444                 case PCA_BAD_QACCESS_VF:
2445                         estr = "PCA_BAD_QACCESS_VF"; break;
2446                 case PCA_MALICIOUS_REQ:
2447                         estr = "PCA_MALICIOUS_REQ"; break;
2448                 case PCA_POISONED_TLP:
2449                         estr = "PCA_POISONED_TLP"; break;
2450                 case PCA_TLP_ABORT:
2451                         estr = "PCA_TLP_ABORT"; break;
2452                 default:
2453                         goto error;
2454                 }
2455                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2456                         estr, fault.func ? "VF" : "PF", fault.func,
2457                         fault.address, fault.specinfo);
2458         }
2459
2460         /* Process THI fault */
2461         if (eicr & FM10K_EICR_THI_FAULT) {
2462                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2463                 if (err)
2464                         goto error;
2465                 switch (fault.type) {
2466                 case THI_NO_FAULT:
2467                         estr = "THI_NO_FAULT"; break;
2468                 case THI_MAL_DIS_Q_FAULT:
2469                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2470                 default:
2471                         goto error;
2472                 }
2473                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2474                         estr, fault.func ? "VF" : "PF", fault.func,
2475                         fault.address, fault.specinfo);
2476         }
2477
2478         /* Process FUM fault */
2479         if (eicr & FM10K_EICR_FUM_FAULT) {
2480                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2481                 if (err)
2482                         goto error;
2483                 switch (fault.type) {
2484                 case FUM_NO_FAULT:
2485                         estr = "FUM_NO_FAULT"; break;
2486                 case FUM_UNMAPPED_ADDR:
2487                         estr = "FUM_UNMAPPED_ADDR"; break;
2488                 case FUM_POISONED_TLP:
2489                         estr = "FUM_POISONED_TLP"; break;
2490                 case FUM_BAD_VF_QACCESS:
2491                         estr = "FUM_BAD_VF_QACCESS"; break;
2492                 case FUM_ADD_DECODE_ERR:
2493                         estr = "FUM_ADD_DECODE_ERR"; break;
2494                 case FUM_RO_ERROR:
2495                         estr = "FUM_RO_ERROR"; break;
2496                 case FUM_QPRC_CRC_ERROR:
2497                         estr = "FUM_QPRC_CRC_ERROR"; break;
2498                 case FUM_CSR_TIMEOUT:
2499                         estr = "FUM_CSR_TIMEOUT"; break;
2500                 case FUM_INVALID_TYPE:
2501                         estr = "FUM_INVALID_TYPE"; break;
2502                 case FUM_INVALID_LENGTH:
2503                         estr = "FUM_INVALID_LENGTH"; break;
2504                 case FUM_INVALID_BE:
2505                         estr = "FUM_INVALID_BE"; break;
2506                 case FUM_INVALID_ALIGN:
2507                         estr = "FUM_INVALID_ALIGN"; break;
2508                 default:
2509                         goto error;
2510                 }
2511                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2512                         estr, fault.func ? "VF" : "PF", fault.func,
2513                         fault.address, fault.specinfo);
2514         }
2515
2516         return 0;
2517 error:
2518         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2519         return err;
2520 }
2521
2522 /**
2523  * PF interrupt handler triggered by NIC for handling specific interrupt.
2524  *
2525  * @param handle
2526  *  Pointer to interrupt handle.
2527  * @param param
2528  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2529  *
2530  * @return
2531  *  void
2532  */
2533 static void
2534 fm10k_dev_interrupt_handler_pf(
2535                         __rte_unused struct rte_intr_handle *handle,
2536                         void *param)
2537 {
2538         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2539         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2540         uint32_t cause, status;
2541
2542         if (hw->mac.type != fm10k_mac_pf)
2543                 return;
2544
2545         cause = FM10K_READ_REG(hw, FM10K_EICR);
2546
2547         /* Handle PCI fault cases */
2548         if (cause & FM10K_EICR_FAULT_MASK) {
2549                 PMD_INIT_LOG(ERR, "INT: find fault!");
2550                 fm10k_dev_handle_fault(hw, cause);
2551         }
2552
2553         /* Handle switch up/down */
2554         if (cause & FM10K_EICR_SWITCHNOTREADY)
2555                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2556
2557         if (cause & FM10K_EICR_SWITCHREADY)
2558                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2559
2560         /* Handle mailbox message */
2561         fm10k_mbx_lock(hw);
2562         hw->mbx.ops.process(hw, &hw->mbx);
2563         fm10k_mbx_unlock(hw);
2564
2565         /* Handle SRAM error */
2566         if (cause & FM10K_EICR_SRAMERROR) {
2567                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2568
2569                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2570                 /* Write to clear pending bits */
2571                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2572
2573                 /* Todo: print out error message after shared code  updates */
2574         }
2575
2576         /* Clear these 3 events if having any */
2577         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2578                  FM10K_EICR_SWITCHREADY;
2579         if (cause)
2580                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2581
2582         /* Re-enable interrupt from device side */
2583         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2584                                         FM10K_ITR_MASK_CLEAR);
2585         /* Re-enable interrupt from host side */
2586         rte_intr_enable(&(dev->pci_dev->intr_handle));
2587 }
2588
2589 /**
2590  * VF interrupt handler triggered by NIC for handling specific interrupt.
2591  *
2592  * @param handle
2593  *  Pointer to interrupt handle.
2594  * @param param
2595  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2596  *
2597  * @return
2598  *  void
2599  */
2600 static void
2601 fm10k_dev_interrupt_handler_vf(
2602                         __rte_unused struct rte_intr_handle *handle,
2603                         void *param)
2604 {
2605         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2606         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2607
2608         if (hw->mac.type != fm10k_mac_vf)
2609                 return;
2610
2611         /* Handle mailbox message if lock is acquired */
2612         fm10k_mbx_lock(hw);
2613         hw->mbx.ops.process(hw, &hw->mbx);
2614         fm10k_mbx_unlock(hw);
2615
2616         /* Re-enable interrupt from device side */
2617         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2618                                         FM10K_ITR_MASK_CLEAR);
2619         /* Re-enable interrupt from host side */
2620         rte_intr_enable(&(dev->pci_dev->intr_handle));
2621 }
2622
2623 /* Mailbox message handler in VF */
2624 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2625         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2626         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2627         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2628         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2629 };
2630
2631 static int
2632 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2633 {
2634         int err = 0;
2635
2636         /* Initialize mailbox lock */
2637         fm10k_mbx_initlock(hw);
2638
2639         /* Replace default message handler with new ones */
2640         if (hw->mac.type == fm10k_mac_vf)
2641                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2642
2643         if (err) {
2644                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2645                                 err);
2646                 return err;
2647         }
2648         /* Connect to SM for PF device or PF for VF device */
2649         return hw->mbx.ops.connect(hw, &hw->mbx);
2650 }
2651
2652 static void
2653 fm10k_close_mbx_service(struct fm10k_hw *hw)
2654 {
2655         /* Disconnect from SM for PF device or PF for VF device */
2656         hw->mbx.ops.disconnect(hw, &hw->mbx);
2657 }
2658
2659 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2660         .dev_configure          = fm10k_dev_configure,
2661         .dev_start              = fm10k_dev_start,
2662         .dev_stop               = fm10k_dev_stop,
2663         .dev_close              = fm10k_dev_close,
2664         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2665         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2666         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2667         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2668         .stats_get              = fm10k_stats_get,
2669         .xstats_get             = fm10k_xstats_get,
2670         .xstats_get_names       = fm10k_xstats_get_names,
2671         .stats_reset            = fm10k_stats_reset,
2672         .xstats_reset           = fm10k_stats_reset,
2673         .link_update            = fm10k_link_update,
2674         .dev_infos_get          = fm10k_dev_infos_get,
2675         .dev_supported_ptypes_get = fm10k_dev_supported_ptypes_get,
2676         .vlan_filter_set        = fm10k_vlan_filter_set,
2677         .vlan_offload_set       = fm10k_vlan_offload_set,
2678         .mac_addr_add           = fm10k_macaddr_add,
2679         .mac_addr_remove        = fm10k_macaddr_remove,
2680         .rx_queue_start         = fm10k_dev_rx_queue_start,
2681         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2682         .tx_queue_start         = fm10k_dev_tx_queue_start,
2683         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2684         .rx_queue_setup         = fm10k_rx_queue_setup,
2685         .rx_queue_release       = fm10k_rx_queue_release,
2686         .tx_queue_setup         = fm10k_tx_queue_setup,
2687         .tx_queue_release       = fm10k_tx_queue_release,
2688         .rx_descriptor_done     = fm10k_dev_rx_descriptor_done,
2689         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2690         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2691         .reta_update            = fm10k_reta_update,
2692         .reta_query             = fm10k_reta_query,
2693         .rss_hash_update        = fm10k_rss_hash_update,
2694         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2695 };
2696
2697 static int ftag_check_handler(__rte_unused const char *key,
2698                 const char *value, __rte_unused void *opaque)
2699 {
2700         if (strcmp(value, "1"))
2701                 return -1;
2702
2703         return 0;
2704 }
2705
2706 static int
2707 fm10k_check_ftag(struct rte_devargs *devargs)
2708 {
2709         struct rte_kvargs *kvlist;
2710         const char *ftag_key = "enable_ftag";
2711
2712         if (devargs == NULL)
2713                 return 0;
2714
2715         kvlist = rte_kvargs_parse(devargs->args, NULL);
2716         if (kvlist == NULL)
2717                 return 0;
2718
2719         if (!rte_kvargs_count(kvlist, ftag_key)) {
2720                 rte_kvargs_free(kvlist);
2721                 return 0;
2722         }
2723         /* FTAG is enabled when there's key-value pair: enable_ftag=1 */
2724         if (rte_kvargs_process(kvlist, ftag_key,
2725                                 ftag_check_handler, NULL) < 0) {
2726                 rte_kvargs_free(kvlist);
2727                 return 0;
2728         }
2729         rte_kvargs_free(kvlist);
2730
2731         return 1;
2732 }
2733
2734 static void __attribute__((cold))
2735 fm10k_set_tx_function(struct rte_eth_dev *dev)
2736 {
2737         struct fm10k_tx_queue *txq;
2738         int i;
2739         int use_sse = 1;
2740         uint16_t tx_ftag_en = 0;
2741
2742         if (fm10k_check_ftag(dev->pci_dev->device.devargs))
2743                 tx_ftag_en = 1;
2744
2745         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2746                 txq = dev->data->tx_queues[i];
2747                 txq->tx_ftag_en = tx_ftag_en;
2748                 /* Check if Vector Tx is satisfied */
2749                 if (fm10k_tx_vec_condition_check(txq))
2750                         use_sse = 0;
2751         }
2752
2753         if (use_sse) {
2754                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2755                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2756                         txq = dev->data->tx_queues[i];
2757                         fm10k_txq_vec_setup(txq);
2758                 }
2759                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2760         } else {
2761                 dev->tx_pkt_burst = fm10k_xmit_pkts;
2762                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2763         }
2764 }
2765
2766 static void __attribute__((cold))
2767 fm10k_set_rx_function(struct rte_eth_dev *dev)
2768 {
2769         struct fm10k_dev_info *dev_info =
2770                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2771         uint16_t i, rx_using_sse;
2772         uint16_t rx_ftag_en = 0;
2773
2774         if (fm10k_check_ftag(dev->pci_dev->device.devargs))
2775                 rx_ftag_en = 1;
2776
2777         /* In order to allow Vector Rx there are a few configuration
2778          * conditions to be met.
2779          */
2780         if (!fm10k_rx_vec_condition_check(dev) &&
2781                         dev_info->rx_vec_allowed && !rx_ftag_en) {
2782                 if (dev->data->scattered_rx)
2783                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
2784                 else
2785                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
2786         } else if (dev->data->scattered_rx)
2787                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
2788         else
2789                 dev->rx_pkt_burst = fm10k_recv_pkts;
2790
2791         rx_using_sse =
2792                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
2793                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
2794
2795         if (rx_using_sse)
2796                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
2797         else
2798                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
2799
2800         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2801                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
2802
2803                 rxq->rx_using_sse = rx_using_sse;
2804                 rxq->rx_ftag_en = rx_ftag_en;
2805         }
2806 }
2807
2808 static void
2809 fm10k_params_init(struct rte_eth_dev *dev)
2810 {
2811         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2812         struct fm10k_dev_info *info =
2813                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2814
2815         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
2816          * there is no way to get link status without reading BAR4.  Until this
2817          * works, assume we have maximum bandwidth.
2818          * @todo - fix bus info
2819          */
2820         hw->bus_caps.speed = fm10k_bus_speed_8000;
2821         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
2822         hw->bus_caps.payload = fm10k_bus_payload_512;
2823         hw->bus.speed = fm10k_bus_speed_8000;
2824         hw->bus.width = fm10k_bus_width_pcie_x8;
2825         hw->bus.payload = fm10k_bus_payload_256;
2826
2827         info->rx_vec_allowed = true;
2828 }
2829
2830 static int
2831 eth_fm10k_dev_init(struct rte_eth_dev *dev)
2832 {
2833         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2834         int diag, i;
2835         struct fm10k_macvlan_filter_info *macvlan;
2836
2837         PMD_INIT_FUNC_TRACE();
2838
2839         dev->dev_ops = &fm10k_eth_dev_ops;
2840         dev->rx_pkt_burst = &fm10k_recv_pkts;
2841         dev->tx_pkt_burst = &fm10k_xmit_pkts;
2842
2843         /* only initialize in the primary process */
2844         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2845                 return 0;
2846
2847         rte_eth_copy_pci_info(dev, dev->pci_dev);
2848
2849         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
2850         memset(macvlan, 0, sizeof(*macvlan));
2851         /* Vendor and Device ID need to be set before init of shared code */
2852         memset(hw, 0, sizeof(*hw));
2853         hw->device_id = dev->pci_dev->id.device_id;
2854         hw->vendor_id = dev->pci_dev->id.vendor_id;
2855         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
2856         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
2857         hw->revision_id = 0;
2858         hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
2859         if (hw->hw_addr == NULL) {
2860                 PMD_INIT_LOG(ERR, "Bad mem resource."
2861                         " Try to blacklist unused devices.");
2862                 return -EIO;
2863         }
2864
2865         /* Store fm10k_adapter pointer */
2866         hw->back = dev->data->dev_private;
2867
2868         /* Initialize the shared code */
2869         diag = fm10k_init_shared_code(hw);
2870         if (diag != FM10K_SUCCESS) {
2871                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
2872                 return -EIO;
2873         }
2874
2875         /* Initialize parameters */
2876         fm10k_params_init(dev);
2877
2878         /* Initialize the hw */
2879         diag = fm10k_init_hw(hw);
2880         if (diag != FM10K_SUCCESS) {
2881                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
2882                 return -EIO;
2883         }
2884
2885         /* Initialize MAC address(es) */
2886         dev->data->mac_addrs = rte_zmalloc("fm10k",
2887                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
2888         if (dev->data->mac_addrs == NULL) {
2889                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
2890                 return -ENOMEM;
2891         }
2892
2893         diag = fm10k_read_mac_addr(hw);
2894
2895         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2896                         &dev->data->mac_addrs[0]);
2897
2898         if (diag != FM10K_SUCCESS ||
2899                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
2900
2901                 /* Generate a random addr */
2902                 eth_random_addr(hw->mac.addr);
2903                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
2904                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2905                 &dev->data->mac_addrs[0]);
2906         }
2907
2908         /* Reset the hw statistics */
2909         fm10k_stats_reset(dev);
2910
2911         /* Reset the hw */
2912         diag = fm10k_reset_hw(hw);
2913         if (diag != FM10K_SUCCESS) {
2914                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
2915                 return -EIO;
2916         }
2917
2918         /* Setup mailbox service */
2919         diag = fm10k_setup_mbx_service(hw);
2920         if (diag != FM10K_SUCCESS) {
2921                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
2922                 return -EIO;
2923         }
2924
2925         /*PF/VF has different interrupt handling mechanism */
2926         if (hw->mac.type == fm10k_mac_pf) {
2927                 /* register callback func to eal lib */
2928                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2929                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2930
2931                 /* enable MISC interrupt */
2932                 fm10k_dev_enable_intr_pf(dev);
2933         } else { /* VF */
2934                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2935                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2936
2937                 fm10k_dev_enable_intr_vf(dev);
2938         }
2939
2940         /* Enable intr after callback registered */
2941         rte_intr_enable(&(dev->pci_dev->intr_handle));
2942
2943         hw->mac.ops.update_int_moderator(hw);
2944
2945         /* Make sure Switch Manager is ready before going forward. */
2946         if (hw->mac.type == fm10k_mac_pf) {
2947                 int switch_ready = 0;
2948
2949                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
2950                         fm10k_mbx_lock(hw);
2951                         hw->mac.ops.get_host_state(hw, &switch_ready);
2952                         fm10k_mbx_unlock(hw);
2953                         if (switch_ready)
2954                                 break;
2955                         /* Delay some time to acquire async LPORT_MAP info. */
2956                         rte_delay_us(WAIT_SWITCH_MSG_US);
2957                 }
2958
2959                 if (switch_ready == 0) {
2960                         PMD_INIT_LOG(ERR, "switch is not ready");
2961                         return -1;
2962                 }
2963         }
2964
2965         /*
2966          * Below function will trigger operations on mailbox, acquire lock to
2967          * avoid race condition from interrupt handler. Operations on mailbox
2968          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
2969          * will handle and generate an interrupt to our side. Then,  FIFO in
2970          * mailbox will be touched.
2971          */
2972         fm10k_mbx_lock(hw);
2973         /* Enable port first */
2974         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2975                                         MAX_LPORT_NUM, 1);
2976
2977         /* Set unicast mode by default. App can change to other mode in other
2978          * API func.
2979          */
2980         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
2981                                         FM10K_XCAST_MODE_NONE);
2982
2983         fm10k_mbx_unlock(hw);
2984
2985         /* Make sure default VID is ready before going forward. */
2986         if (hw->mac.type == fm10k_mac_pf) {
2987                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
2988                         if (hw->mac.default_vid)
2989                                 break;
2990                         /* Delay some time to acquire async port VLAN info. */
2991                         rte_delay_us(WAIT_SWITCH_MSG_US);
2992                 }
2993
2994                 if (!hw->mac.default_vid) {
2995                         PMD_INIT_LOG(ERR, "default VID is not ready");
2996                         return -1;
2997                 }
2998         }
2999
3000         /* Add default mac address */
3001         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
3002                 MAIN_VSI_POOL_NUMBER);
3003
3004         return 0;
3005 }
3006
3007 static int
3008 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
3009 {
3010         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3011
3012         PMD_INIT_FUNC_TRACE();
3013
3014         /* only uninitialize in the primary process */
3015         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3016                 return 0;
3017
3018         /* safe to close dev here */
3019         fm10k_dev_close(dev);
3020
3021         dev->dev_ops = NULL;
3022         dev->rx_pkt_burst = NULL;
3023         dev->tx_pkt_burst = NULL;
3024
3025         /* disable uio/vfio intr */
3026         rte_intr_disable(&(dev->pci_dev->intr_handle));
3027
3028         /*PF/VF has different interrupt handling mechanism */
3029         if (hw->mac.type == fm10k_mac_pf) {
3030                 /* disable interrupt */
3031                 fm10k_dev_disable_intr_pf(dev);
3032
3033                 /* unregister callback func to eal lib */
3034                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
3035                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3036         } else {
3037                 /* disable interrupt */
3038                 fm10k_dev_disable_intr_vf(dev);
3039
3040                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
3041                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3042         }
3043
3044         /* free mac memory */
3045         if (dev->data->mac_addrs) {
3046                 rte_free(dev->data->mac_addrs);
3047                 dev->data->mac_addrs = NULL;
3048         }
3049
3050         memset(hw, 0, sizeof(*hw));
3051
3052         return 0;
3053 }
3054
3055 /*
3056  * The set of PCI devices this driver supports. This driver will enable both PF
3057  * and SRIOV-VF devices.
3058  */
3059 static const struct rte_pci_id pci_id_fm10k_map[] = {
3060         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_PF) },
3061         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_SDI_FM10420_QDA2) },
3062         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_VF) },
3063         { .vendor_id = 0, /* sentinel */ },
3064 };
3065
3066 static struct eth_driver rte_pmd_fm10k = {
3067         .pci_drv = {
3068                 .id_table = pci_id_fm10k_map,
3069                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
3070                         RTE_PCI_DRV_DETACHABLE,
3071                 .probe = rte_eth_dev_pci_probe,
3072                 .remove = rte_eth_dev_pci_remove,
3073         },
3074         .eth_dev_init = eth_fm10k_dev_init,
3075         .eth_dev_uninit = eth_fm10k_dev_uninit,
3076         .dev_private_size = sizeof(struct fm10k_adapter),
3077 };
3078
3079 RTE_PMD_REGISTER_PCI(net_fm10k, rte_pmd_fm10k.pci_drv);
3080 RTE_PMD_REGISTER_PCI_TABLE(net_fm10k, pci_id_fm10k_map);