drivers/net/mrvl/mrvl_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2017 Marvell International Ltd.
   5  *   Copyright(c) 2017 Semihalf.
   6  *   All rights reserved.
   7  *
   8  *   Redistribution and use in source and binary forms, with or without
   9  *   modification, are permitted provided that the following conditions
  10  *   are met:
  11  *
  12  *     * Redistributions of source code must retain the above copyright
  13  *       notice, this list of conditions and the following disclaimer.
  14  *     * Redistributions in binary form must reproduce the above copyright
  15  *       notice, this list of conditions and the following disclaimer in
  16  *       the documentation and/or other materials provided with the
  17  *       distribution.
  18  *     * Neither the name of Semihalf nor the names of its
  19  *       contributors may be used to endorse or promote products derived
  20  *       from this software without specific prior written permission.
  21  *
  22  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  23  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  24  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  25  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  26  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  27  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  28  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  29  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  30  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  31  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  32  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  33  */
  34
  35 #include <rte_ethdev.h>
  36 #include <rte_kvargs.h>
  37 #include <rte_log.h>
  38 #include <rte_malloc.h>
  39 #include <rte_bus_vdev.h>
  40
  41 /* Unluckily, container_of is defined by both DPDK and MUSDK,
  42  * we'll declare only one version.
  43  *
  44  * Note that it is not used in this PMD anyway.
  45  */
  46 #ifdef container_of
  47 #undef container_of
  48 #endif
  49
  50 #include <drivers/mv_pp2.h>
  51 #include <drivers/mv_pp2_bpool.h>
  52 #include <drivers/mv_pp2_hif.h>
  53
  54 #include <fcntl.h>
  55 #include <linux/ethtool.h>
  56 #include <linux/sockios.h>
  57 #include <net/if.h>
  58 #include <net/if_arp.h>
  59 #include <sys/ioctl.h>
  60 #include <sys/socket.h>
  61 #include <sys/stat.h>
  62 #include <sys/types.h>
  63
  64 #include "mrvl_ethdev.h"
  65 #include "mrvl_qos.h"
  66
  67 /* bitmask with reserved hifs */
  68 #define MRVL_MUSDK_HIFS_RESERVED 0x0F
  69 /* bitmask with reserved bpools */
  70 #define MRVL_MUSDK_BPOOLS_RESERVED 0x07
  71 /* bitmask with reserved kernel RSS tables */
  72 #define MRVL_MUSDK_RSS_RESERVED 0x01
  73 /* maximum number of available hifs */
  74 #define MRVL_MUSDK_HIFS_MAX 9
  75
  76 /* prefetch shift */
  77 #define MRVL_MUSDK_PREFETCH_SHIFT 2
  78
  79 /* TCAM has 25 entries reserved for uc/mc filter entries */
  80 #define MRVL_MAC_ADDRS_MAX 25
  81 #define MRVL_MATCH_LEN 16
  82 #define MRVL_PKT_EFFEC_OFFS (MRVL_PKT_OFFS + MV_MH_SIZE)
  83 /* Maximum allowable packet size */
  84 #define MRVL_PKT_SIZE_MAX (10240 - MV_MH_SIZE)
  85
  86 #define MRVL_IFACE_NAME_ARG "iface"
  87 #define MRVL_CFG_ARG "cfg"
  88
  89 #define MRVL_BURST_SIZE 64
  90
  91 #define MRVL_ARP_LENGTH 28
  92
  93 #define MRVL_COOKIE_ADDR_INVALID ~0ULL
  94
  95 #define MRVL_COOKIE_HIGH_ADDR_SHIFT     (sizeof(pp2_cookie_t) * 8)
  96 #define MRVL_COOKIE_HIGH_ADDR_MASK      (~0ULL << MRVL_COOKIE_HIGH_ADDR_SHIFT)
  97
  98 /* Memory size (in bytes) for MUSDK dma buffers */
  99 #define MRVL_MUSDK_DMA_MEMSIZE 41943040
 100
 101 static const char * const valid_args[] = {
 102         MRVL_IFACE_NAME_ARG,
 103         MRVL_CFG_ARG,
 104         NULL
 105 };
 106
 107 static int used_hifs = MRVL_MUSDK_HIFS_RESERVED;
 108 static struct pp2_hif *hifs[RTE_MAX_LCORE];
 109 static int used_bpools[PP2_NUM_PKT_PROC] = {
 110         MRVL_MUSDK_BPOOLS_RESERVED,
 111         MRVL_MUSDK_BPOOLS_RESERVED
 112 };
 113
 114 struct pp2_bpool *mrvl_port_to_bpool_lookup[RTE_MAX_ETHPORTS];
 115 int mrvl_port_bpool_size[PP2_NUM_PKT_PROC][PP2_BPOOL_NUM_POOLS][RTE_MAX_LCORE];
 116 uint64_t cookie_addr_high = MRVL_COOKIE_ADDR_INVALID;
 117
 118 struct mrvl_ifnames {
 119         const char *names[PP2_NUM_ETH_PPIO * PP2_NUM_PKT_PROC];
 120         int idx;
 121 };
 122
 123 /*
 124  * To use buffer harvesting based on loopback port shadow queue structure
 125  * was introduced for buffers information bookkeeping.
 126  *
 127  * Before sending the packet, related buffer information (pp2_buff_inf) is
 128  * stored in shadow queue. After packet is transmitted no longer used
 129  * packet buffer is released back to it's original hardware pool,
 130  * on condition it originated from interface.
 131  * In case it  was generated by application itself i.e: mbuf->port field is
 132  * 0xff then its released to software mempool.
 133  */
 134 struct mrvl_shadow_txq {
 135         int head;           /* write index - used when sending buffers */
 136         int tail;           /* read index - used when releasing buffers */
 137         u16 size;           /* queue occupied size */
 138         u16 num_to_release; /* number of buffers sent, that can be released */
 139         struct buff_release_entry ent[MRVL_PP2_TX_SHADOWQ_SIZE]; /* q entries */
 140 };
 141
 142 struct mrvl_rxq {
 143         struct mrvl_priv *priv;
 144         struct rte_mempool *mp;
 145         int queue_id;
 146         int port_id;
 147         int cksum_enabled;
 148         uint64_t bytes_recv;
 149         uint64_t drop_mac;
 150 };
 151
 152 struct mrvl_txq {
 153         struct mrvl_priv *priv;
 154         int queue_id;
 155         int port_id;
 156         uint64_t bytes_sent;
 157         struct mrvl_shadow_txq shadow_txqs[RTE_MAX_LCORE];
 158 };
 159
 160 static int mrvl_lcore_first;
 161 static int mrvl_lcore_last;
 162 static int mrvl_dev_num;
 163
 164 static int mrvl_fill_bpool(struct mrvl_rxq *rxq, int num);
 165 static inline void mrvl_free_sent_buffers(struct pp2_ppio *ppio,
 166                         struct pp2_hif *hif, unsigned int core_id,
 167                         struct mrvl_shadow_txq *sq, int qid, int force);
 168
 169 static inline int
 170 mrvl_get_bpool_size(int pp2_id, int pool_id)
 171 {
 172         int i;
 173         int size = 0;
 174
 175         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++)
 176                 size += mrvl_port_bpool_size[pp2_id][pool_id][i];
 177
 178         return size;
 179 }
 180
 181 static inline int
 182 mrvl_reserve_bit(int *bitmap, int max)
 183 {
 184         int n = sizeof(*bitmap) * 8 - __builtin_clz(*bitmap);
 185
 186         if (n >= max)
 187                 return -1;
 188
 189         *bitmap |= 1 << n;
 190
 191         return n;
 192 }
 193
 194 static int
 195 mrvl_init_hif(int core_id)
 196 {
 197         struct pp2_hif_params params;
 198         char match[MRVL_MATCH_LEN];
 199         int ret;
 200
 201         ret = mrvl_reserve_bit(&used_hifs, MRVL_MUSDK_HIFS_MAX);
 202         if (ret < 0) {
 203                 RTE_LOG(ERR, PMD, "Failed to allocate hif %d\n", core_id);
 204                 return ret;
 205         }
 206
 207         snprintf(match, sizeof(match), "hif-%d", ret);
 208         memset(&params, 0, sizeof(params));
 209         params.match = match;
 210         params.out_size = MRVL_PP2_AGGR_TXQD_MAX;
 211         ret = pp2_hif_init(&params, &hifs[core_id]);
 212         if (ret) {
 213                 RTE_LOG(ERR, PMD, "Failed to initialize hif %d\n", core_id);
 214                 return ret;
 215         }
 216
 217         return 0;
 218 }
 219
 220 static inline struct pp2_hif*
 221 mrvl_get_hif(struct mrvl_priv *priv, int core_id)
 222 {
 223         int ret;
 224
 225         if (likely(hifs[core_id] != NULL))
 226                 return hifs[core_id];
 227
 228         rte_spinlock_lock(&priv->lock);
 229
 230         ret = mrvl_init_hif(core_id);
 231         if (ret < 0) {
 232                 RTE_LOG(ERR, PMD, "Failed to allocate hif %d\n", core_id);
 233                 goto out;
 234         }
 235
 236         if (core_id < mrvl_lcore_first)
 237                 mrvl_lcore_first = core_id;
 238
 239         if (core_id > mrvl_lcore_last)
 240                 mrvl_lcore_last = core_id;
 241 out:
 242         rte_spinlock_unlock(&priv->lock);
 243
 244         return hifs[core_id];
 245 }
 246
 247 /**
 248  * Configure rss based on dpdk rss configuration.
 249  *
 250  * @param priv
 251  *   Pointer to private structure.
 252  * @param rss_conf
 253  *   Pointer to RSS configuration.
 254  *
 255  * @return
 256  *   0 on success, negative error value otherwise.
 257  */
 258 static int
 259 mrvl_configure_rss(struct mrvl_priv *priv, struct rte_eth_rss_conf *rss_conf)
 260 {
 261         if (rss_conf->rss_key)
 262                 RTE_LOG(WARNING, PMD, "Changing hash key is not supported\n");
 263
 264         if (rss_conf->rss_hf == 0) {
 265                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 266         } else if (rss_conf->rss_hf & ETH_RSS_IPV4) {
 267                 priv->ppio_params.inqs_params.hash_type =
 268                         PP2_PPIO_HASH_T_2_TUPLE;
 269         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) {
 270                 priv->ppio_params.inqs_params.hash_type =
 271                         PP2_PPIO_HASH_T_5_TUPLE;
 272                 priv->rss_hf_tcp = 1;
 273         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
 274                 priv->ppio_params.inqs_params.hash_type =
 275                         PP2_PPIO_HASH_T_5_TUPLE;
 276                 priv->rss_hf_tcp = 0;
 277         } else {
 278                 return -EINVAL;
 279         }
 280
 281         return 0;
 282 }
 283
 284 /**
 285  * Ethernet device configuration.
 286  *
 287  * Prepare the driver for a given number of TX and RX queues and
 288  * configure RSS.
 289  *
 290  * @param dev
 291  *   Pointer to Ethernet device structure.
 292  *
 293  * @return
 294  *   0 on success, negative error value otherwise.
 295  */
 296 static int
 297 mrvl_dev_configure(struct rte_eth_dev *dev)
 298 {
 299         struct mrvl_priv *priv = dev->data->dev_private;
 300         int ret;
 301
 302         if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_NONE &&
 303             dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) {
 304                 RTE_LOG(INFO, PMD, "Unsupported rx multi queue mode %d\n",
 305                         dev->data->dev_conf.rxmode.mq_mode);
 306                 return -EINVAL;
 307         }
 308
 309         if (!dev->data->dev_conf.rxmode.hw_strip_crc) {
 310                 RTE_LOG(INFO, PMD,
 311                         "L2 CRC stripping is always enabled in hw\n");
 312                 dev->data->dev_conf.rxmode.hw_strip_crc = 1;
 313         }
 314
 315         if (dev->data->dev_conf.rxmode.hw_vlan_strip) {
 316                 RTE_LOG(INFO, PMD, "VLAN stripping not supported\n");
 317                 return -EINVAL;
 318         }
 319
 320         if (dev->data->dev_conf.rxmode.split_hdr_size) {
 321                 RTE_LOG(INFO, PMD, "Split headers not supported\n");
 322                 return -EINVAL;
 323         }
 324
 325         if (dev->data->dev_conf.rxmode.enable_scatter) {
 326                 RTE_LOG(INFO, PMD, "RX Scatter/Gather not supported\n");
 327                 return -EINVAL;
 328         }
 329
 330         if (dev->data->dev_conf.rxmode.enable_lro) {
 331                 RTE_LOG(INFO, PMD, "LRO not supported\n");
 332                 return -EINVAL;
 333         }
 334
 335         if (dev->data->dev_conf.rxmode.jumbo_frame)
 336                 dev->data->mtu = dev->data->dev_conf.rxmode.max_rx_pkt_len -
 337                                  ETHER_HDR_LEN - ETHER_CRC_LEN;
 338
 339         ret = mrvl_configure_rxqs(priv, dev->data->port_id,
 340                                   dev->data->nb_rx_queues);
 341         if (ret < 0)
 342                 return ret;
 343
 344         priv->ppio_params.outqs_params.num_outqs = dev->data->nb_tx_queues;
 345         priv->ppio_params.maintain_stats = 1;
 346         priv->nb_rx_queues = dev->data->nb_rx_queues;
 347
 348         if (dev->data->nb_rx_queues == 1 &&
 349             dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) {
 350                 RTE_LOG(WARNING, PMD, "Disabling hash for 1 rx queue\n");
 351                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 352
 353                 return 0;
 354         }
 355
 356         return mrvl_configure_rss(priv,
 357                                   &dev->data->dev_conf.rx_adv_conf.rss_conf);
 358 }
 359
 360 /**
 361  * DPDK callback to change the MTU.
 362  *
 363  * Setting the MTU affects hardware MRU (packets larger than the MRU
 364  * will be dropped).
 365  *
 366  * @param dev
 367  *   Pointer to Ethernet device structure.
 368  * @param mtu
 369  *   New MTU.
 370  *
 371  * @return
 372  *   0 on success, negative error value otherwise.
 373  */
 374 static int
 375 mrvl_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 376 {
 377         struct mrvl_priv *priv = dev->data->dev_private;
 378         /* extra MV_MH_SIZE bytes are required for Marvell tag */
 379         uint16_t mru = mtu + MV_MH_SIZE + ETHER_HDR_LEN + ETHER_CRC_LEN;
 380         int ret;
 381
 382         if (mtu < ETHER_MIN_MTU || mru > MRVL_PKT_SIZE_MAX)
 383                 return -EINVAL;
 384
 385         ret = pp2_ppio_set_mru(priv->ppio, mru);
 386         if (ret)
 387                 return ret;
 388
 389         return pp2_ppio_set_mtu(priv->ppio, mtu);
 390 }
 391
 392 /**
 393  * DPDK callback to bring the link up.
 394  *
 395  * @param dev
 396  *   Pointer to Ethernet device structure.
 397  *
 398  * @return
 399  *   0 on success, negative error value otherwise.
 400  */
 401 static int
 402 mrvl_dev_set_link_up(struct rte_eth_dev *dev)
 403 {
 404         struct mrvl_priv *priv = dev->data->dev_private;
 405         int ret;
 406
 407         ret = pp2_ppio_enable(priv->ppio);
 408         if (ret)
 409                 return ret;
 410
 411         /*
 412          * mtu/mru can be updated if pp2_ppio_enable() was called at least once
 413          * as pp2_ppio_enable() changes port->t_mode from default 0 to
 414          * PP2_TRAFFIC_INGRESS_EGRESS.
 415          *
 416          * Set mtu to default DPDK value here.
 417          */
 418         ret = mrvl_mtu_set(dev, dev->data->mtu);
 419         if (ret)
 420                 pp2_ppio_disable(priv->ppio);
 421
 422         dev->data->dev_link.link_status = ETH_LINK_UP;
 423
 424         return ret;
 425 }
 426
 427 /**
 428  * DPDK callback to bring the link down.
 429  *
 430  * @param dev
 431  *   Pointer to Ethernet device structure.
 432  *
 433  * @return
 434  *   0 on success, negative error value otherwise.
 435  */
 436 static int
 437 mrvl_dev_set_link_down(struct rte_eth_dev *dev)
 438 {
 439         struct mrvl_priv *priv = dev->data->dev_private;
 440         int ret;
 441
 442         ret = pp2_ppio_disable(priv->ppio);
 443         if (ret)
 444                 return ret;
 445
 446         dev->data->dev_link.link_status = ETH_LINK_DOWN;
 447
 448         return ret;
 449 }
 450
 451 /**
 452  * DPDK callback to start the device.
 453  *
 454  * @param dev
 455  *   Pointer to Ethernet device structure.
 456  *
 457  * @return
 458  *   0 on success, negative errno value on failure.
 459  */
 460 static int
 461 mrvl_dev_start(struct rte_eth_dev *dev)
 462 {
 463         struct mrvl_priv *priv = dev->data->dev_private;
 464         char match[MRVL_MATCH_LEN];
 465         int ret = 0, def_init_size;
 466
 467         snprintf(match, sizeof(match), "ppio-%d:%d",
 468                  priv->pp_id, priv->ppio_id);
 469         priv->ppio_params.match = match;
 470
 471         /*
 472          * Calculate the minimum bpool size for refill feature as follows:
 473          * 2 default burst sizes multiply by number of rx queues.
 474          * If the bpool size will be below this value, new buffers will
 475          * be added to the pool.
 476          */
 477         priv->bpool_min_size = priv->nb_rx_queues * MRVL_BURST_SIZE * 2;
 478
 479         /* In case initial bpool size configured in queues setup is
 480          * smaller than minimum size add more buffers
 481          */
 482         def_init_size = priv->bpool_min_size + MRVL_BURST_SIZE * 2;
 483         if (priv->bpool_init_size < def_init_size) {
 484                 int buffs_to_add = def_init_size - priv->bpool_init_size;
 485
 486                 priv->bpool_init_size += buffs_to_add;
 487                 ret = mrvl_fill_bpool(dev->data->rx_queues[0], buffs_to_add);
 488                 if (ret)
 489                         RTE_LOG(ERR, PMD, "Failed to add buffers to bpool\n");
 490         }
 491
 492         /*
 493          * Calculate the maximum bpool size for refill feature as follows:
 494          * maximum number of descriptors in rx queue multiply by number
 495          * of rx queues plus minimum bpool size.
 496          * In case the bpool size will exceed this value, superfluous buffers
 497          * will be removed
 498          */
 499         priv->bpool_max_size = (priv->nb_rx_queues * MRVL_PP2_RXD_MAX) +
 500                                 priv->bpool_min_size;
 501
 502         ret = pp2_ppio_init(&priv->ppio_params, &priv->ppio);
 503         if (ret)
 504                 return ret;
 505
 506         /*
 507          * In case there are some some stale uc/mc mac addresses flush them
 508          * here. It cannot be done during mrvl_dev_close() as port information
 509          * is already gone at that point (due to pp2_ppio_deinit() in
 510          * mrvl_dev_stop()).
 511          */
 512         if (!priv->uc_mc_flushed) {
 513                 ret = pp2_ppio_flush_mac_addrs(priv->ppio, 1, 1);
 514                 if (ret) {
 515                         RTE_LOG(ERR, PMD,
 516                                 "Failed to flush uc/mc filter list\n");
 517                         goto out;
 518                 }
 519                 priv->uc_mc_flushed = 1;
 520         }
 521
 522         if (!priv->vlan_flushed) {
 523                 ret = pp2_ppio_flush_vlan(priv->ppio);
 524                 if (ret) {
 525                         RTE_LOG(ERR, PMD, "Failed to flush vlan list\n");
 526                         /*
 527                          * TODO
 528                          * once pp2_ppio_flush_vlan() is supported jump to out
 529                          * goto out;
 530                          */
 531                 }
 532                 priv->vlan_flushed = 1;
 533         }
 534
 535         /* For default QoS config, don't start classifier. */
 536         if (mrvl_qos_cfg) {
 537                 ret = mrvl_start_qos_mapping(priv);
 538                 if (ret) {
 539                         pp2_ppio_deinit(priv->ppio);
 540                         return ret;
 541                 }
 542         }
 543
 544         ret = mrvl_dev_set_link_up(dev);
 545         if (ret)
 546                 goto out;
 547
 548         return 0;
 549 out:
 550         pp2_ppio_deinit(priv->ppio);
 551         return ret;
 552 }
 553
 554 /**
 555  * Flush receive queues.
 556  *
 557  * @param dev
 558  *   Pointer to Ethernet device structure.
 559  */
 560 static void
 561 mrvl_flush_rx_queues(struct rte_eth_dev *dev)
 562 {
 563         int i;
 564
 565         RTE_LOG(INFO, PMD, "Flushing rx queues\n");
 566         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 567                 int ret, num;
 568
 569                 do {
 570                         struct mrvl_rxq *q = dev->data->rx_queues[i];
 571                         struct pp2_ppio_desc descs[MRVL_PP2_RXD_MAX];
 572
 573                         num = MRVL_PP2_RXD_MAX;
 574                         ret = pp2_ppio_recv(q->priv->ppio,
 575                                             q->priv->rxq_map[q->queue_id].tc,
 576                                             q->priv->rxq_map[q->queue_id].inq,
 577                                             descs, (uint16_t *)&num);
 578                 } while (ret == 0 && num);
 579         }
 580 }
 581
 582 /**
 583  * Flush transmit shadow queues.
 584  *
 585  * @param dev
 586  *   Pointer to Ethernet device structure.
 587  */
 588 static void
 589 mrvl_flush_tx_shadow_queues(struct rte_eth_dev *dev)
 590 {
 591         int i, j;
 592         struct mrvl_txq *txq;
 593
 594         RTE_LOG(INFO, PMD, "Flushing tx shadow queues\n");
 595         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 596                 txq = (struct mrvl_txq *)dev->data->tx_queues[i];
 597
 598                 for (j = 0; j < RTE_MAX_LCORE; j++) {
 599                         struct mrvl_shadow_txq *sq;
 600
 601                         if (!hifs[j])
 602                                 continue;
 603
 604                         sq = &txq->shadow_txqs[j];
 605                         mrvl_free_sent_buffers(txq->priv->ppio,
 606                                 hifs[j], j, sq, txq->queue_id, 1);
 607                         while (sq->tail != sq->head) {
 608                                 uint64_t addr = cookie_addr_high |
 609                                         sq->ent[sq->tail].buff.cookie;
 610                                 rte_pktmbuf_free(
 611                                         (struct rte_mbuf *)addr);
 612                                 sq->tail = (sq->tail + 1) &
 613                                             MRVL_PP2_TX_SHADOWQ_MASK;
 614                         }
 615                         memset(sq, 0, sizeof(*sq));
 616                 }
 617         }
 618 }
 619
 620 /**
 621  * Flush hardware bpool (buffer-pool).
 622  *
 623  * @param dev
 624  *   Pointer to Ethernet device structure.
 625  */
 626 static void
 627 mrvl_flush_bpool(struct rte_eth_dev *dev)
 628 {
 629         struct mrvl_priv *priv = dev->data->dev_private;
 630         struct pp2_hif *hif;
 631         uint32_t num;
 632         int ret;
 633         unsigned int core_id = rte_lcore_id();
 634
 635         if (core_id == LCORE_ID_ANY)
 636                 core_id = 0;
 637
 638         hif = mrvl_get_hif(priv, core_id);
 639
 640         ret = pp2_bpool_get_num_buffs(priv->bpool, &num);
 641         if (ret) {
 642                 RTE_LOG(ERR, PMD, "Failed to get bpool buffers number\n");
 643                 return;
 644         }
 645
 646         while (num--) {
 647                 struct pp2_buff_inf inf;
 648                 uint64_t addr;
 649
 650                 ret = pp2_bpool_get_buff(hif, priv->bpool, &inf);
 651                 if (ret)
 652                         break;
 653
 654                 addr = cookie_addr_high | inf.cookie;
 655                 rte_pktmbuf_free((struct rte_mbuf *)addr);
 656         }
 657 }
 658
 659 /**
 660  * DPDK callback to stop the device.
 661  *
 662  * @param dev
 663  *   Pointer to Ethernet device structure.
 664  */
 665 static void
 666 mrvl_dev_stop(struct rte_eth_dev *dev)
 667 {
 668         struct mrvl_priv *priv = dev->data->dev_private;
 669
 670         mrvl_dev_set_link_down(dev);
 671         mrvl_flush_rx_queues(dev);
 672         mrvl_flush_tx_shadow_queues(dev);
 673         if (priv->qos_tbl) {
 674                 pp2_cls_qos_tbl_deinit(priv->qos_tbl);
 675                 priv->qos_tbl = NULL;
 676         }
 677         if (priv->ppio)
 678                 pp2_ppio_deinit(priv->ppio);
 679         priv->ppio = NULL;
 680 }
 681
 682 /**
 683  * DPDK callback to close the device.
 684  *
 685  * @param dev
 686  *   Pointer to Ethernet device structure.
 687  */
 688 static void
 689 mrvl_dev_close(struct rte_eth_dev *dev)
 690 {
 691         struct mrvl_priv *priv = dev->data->dev_private;
 692         size_t i;
 693
 694         for (i = 0; i < priv->ppio_params.inqs_params.num_tcs; ++i) {
 695                 struct pp2_ppio_tc_params *tc_params =
 696                         &priv->ppio_params.inqs_params.tcs_params[i];
 697
 698                 if (tc_params->inqs_params) {
 699                         rte_free(tc_params->inqs_params);
 700                         tc_params->inqs_params = NULL;
 701                 }
 702         }
 703
 704         mrvl_flush_bpool(dev);
 705 }
 706
 707 /**
 708  * DPDK callback to retrieve physical link information.
 709  *
 710  * @param dev
 711  *   Pointer to Ethernet device structure.
 712  * @param wait_to_complete
 713  *   Wait for request completion (ignored).
 714  *
 715  * @return
 716  *   0 on success, negative error value otherwise.
 717  */
 718 static int
 719 mrvl_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused)
 720 {
 721         /*
 722          * TODO
 723          * once MUSDK provides necessary API use it here
 724          */
 725         struct ethtool_cmd edata;
 726         struct ifreq req;
 727         int ret, fd;
 728
 729         edata.cmd = ETHTOOL_GSET;
 730
 731         strcpy(req.ifr_name, dev->data->name);
 732         req.ifr_data = (void *)&edata;
 733
 734         fd = socket(AF_INET, SOCK_DGRAM, 0);
 735         if (fd == -1)
 736                 return -EFAULT;
 737
 738         ret = ioctl(fd, SIOCETHTOOL, &req);
 739         if (ret == -1) {
 740                 close(fd);
 741                 return -EFAULT;
 742         }
 743
 744         close(fd);
 745
 746         switch (ethtool_cmd_speed(&edata)) {
 747         case SPEED_10:
 748                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10M;
 749                 break;
 750         case SPEED_100:
 751                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_100M;
 752                 break;
 753         case SPEED_1000:
 754                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_1G;
 755                 break;
 756         case SPEED_10000:
 757                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10G;
 758                 break;
 759         default:
 760                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE;
 761         }
 762
 763         dev->data->dev_link.link_duplex = edata.duplex ? ETH_LINK_FULL_DUPLEX :
 764                                                          ETH_LINK_HALF_DUPLEX;
 765         dev->data->dev_link.link_autoneg = edata.autoneg ? ETH_LINK_AUTONEG :
 766                                                            ETH_LINK_FIXED;
 767
 768         return 0;
 769 }
 770
 771 /**
 772  * DPDK callback to enable promiscuous mode.
 773  *
 774  * @param dev
 775  *   Pointer to Ethernet device structure.
 776  */
 777 static void
 778 mrvl_promiscuous_enable(struct rte_eth_dev *dev)
 779 {
 780         struct mrvl_priv *priv = dev->data->dev_private;
 781         int ret;
 782
 783         ret = pp2_ppio_set_uc_promisc(priv->ppio, 1);
 784         if (ret)
 785                 RTE_LOG(ERR, PMD, "Failed to enable promiscuous mode\n");
 786 }
 787
 788 /**
 789  * DPDK callback to enable allmulti mode.
 790  *
 791  * @param dev
 792  *   Pointer to Ethernet device structure.
 793  */
 794 static void
 795 mrvl_allmulticast_enable(struct rte_eth_dev *dev)
 796 {
 797         struct mrvl_priv *priv = dev->data->dev_private;
 798         int ret;
 799
 800         ret = pp2_ppio_set_mc_promisc(priv->ppio, 1);
 801         if (ret)
 802                 RTE_LOG(ERR, PMD, "Failed enable all-multicast mode\n");
 803 }
 804
 805 /**
 806  * DPDK callback to disable promiscuous mode.
 807  *
 808  * @param dev
 809  *   Pointer to Ethernet device structure.
 810  */
 811 static void
 812 mrvl_promiscuous_disable(struct rte_eth_dev *dev)
 813 {
 814         struct mrvl_priv *priv = dev->data->dev_private;
 815         int ret;
 816
 817         ret = pp2_ppio_set_uc_promisc(priv->ppio, 0);
 818         if (ret)
 819                 RTE_LOG(ERR, PMD, "Failed to disable promiscuous mode\n");
 820 }
 821
 822 /**
 823  * DPDK callback to disable allmulticast mode.
 824  *
 825  * @param dev
 826  *   Pointer to Ethernet device structure.
 827  */
 828 static void
 829 mrvl_allmulticast_disable(struct rte_eth_dev *dev)
 830 {
 831         struct mrvl_priv *priv = dev->data->dev_private;
 832         int ret;
 833
 834         ret = pp2_ppio_set_mc_promisc(priv->ppio, 0);
 835         if (ret)
 836                 RTE_LOG(ERR, PMD, "Failed to disable all-multicast mode\n");
 837 }
 838
 839 /**
 840  * DPDK callback to remove a MAC address.
 841  *
 842  * @param dev
 843  *   Pointer to Ethernet device structure.
 844  * @param index
 845  *   MAC address index.
 846  */
 847 static void
 848 mrvl_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
 849 {
 850         struct mrvl_priv *priv = dev->data->dev_private;
 851         char buf[ETHER_ADDR_FMT_SIZE];
 852         int ret;
 853
 854         ret = pp2_ppio_remove_mac_addr(priv->ppio,
 855                                        dev->data->mac_addrs[index].addr_bytes);
 856         if (ret) {
 857                 ether_format_addr(buf, sizeof(buf),
 858                                   &dev->data->mac_addrs[index]);
 859                 RTE_LOG(ERR, PMD, "Failed to remove mac %s\n", buf);
 860         }
 861 }
 862
 863 /**
 864  * DPDK callback to add a MAC address.
 865  *
 866  * @param dev
 867  *   Pointer to Ethernet device structure.
 868  * @param mac_addr
 869  *   MAC address to register.
 870  * @param index
 871  *   MAC address index.
 872  * @param vmdq
 873  *   VMDq pool index to associate address with (unused).
 874  *
 875  * @return
 876  *   0 on success, negative error value otherwise.
 877  */
 878 static int
 879 mrvl_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
 880                   uint32_t index, uint32_t vmdq __rte_unused)
 881 {
 882         struct mrvl_priv *priv = dev->data->dev_private;
 883         char buf[ETHER_ADDR_FMT_SIZE];
 884         int ret;
 885
 886         if (index == 0)
 887                 /* For setting index 0, mrvl_mac_addr_set() should be used.*/
 888                 return -1;
 889
 890         /*
 891          * Maximum number of uc addresses can be tuned via kernel module mvpp2x
 892          * parameter uc_filter_max. Maximum number of mc addresses is then
 893          * MRVL_MAC_ADDRS_MAX - uc_filter_max. Currently it defaults to 4 and
 894          * 21 respectively.
 895          *
 896          * If more than uc_filter_max uc addresses were added to filter list
 897          * then NIC will switch to promiscuous mode automatically.
 898          *
 899          * If more than MRVL_MAC_ADDRS_MAX - uc_filter_max number mc addresses
 900          * were added to filter list then NIC will switch to all-multicast mode
 901          * automatically.
 902          */
 903         ret = pp2_ppio_add_mac_addr(priv->ppio, mac_addr->addr_bytes);
 904         if (ret) {
 905                 ether_format_addr(buf, sizeof(buf), mac_addr);
 906                 RTE_LOG(ERR, PMD, "Failed to add mac %s\n", buf);
 907                 return -1;
 908         }
 909
 910         return 0;
 911 }
 912
 913 /**
 914  * DPDK callback to set the primary MAC address.
 915  *
 916  * @param dev
 917  *   Pointer to Ethernet device structure.
 918  * @param mac_addr
 919  *   MAC address to register.
 920  */
 921 static void
 922 mrvl_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
 923 {
 924         struct mrvl_priv *priv = dev->data->dev_private;
 925
 926         pp2_ppio_set_mac_addr(priv->ppio, mac_addr->addr_bytes);
 927         /*
 928          * TODO
 929          * Port stops sending packets if pp2_ppio_set_mac_addr()
 930          * was called after pp2_ppio_enable(). As a quick fix issue
 931          * enable port once again.
 932          */
 933         pp2_ppio_enable(priv->ppio);
 934 }
 935
 936 /**
 937  * DPDK callback to get device statistics.
 938  *
 939  * @param dev
 940  *   Pointer to Ethernet device structure.
 941  * @param stats
 942  *   Stats structure output buffer.
 943  *
 944  * @return
 945  *   0 on success, negative error value otherwise.
 946  */
 947 static int
 948 mrvl_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 949 {
 950         struct mrvl_priv *priv = dev->data->dev_private;
 951         struct pp2_ppio_statistics ppio_stats;
 952         uint64_t drop_mac = 0;
 953         unsigned int i, idx, ret;
 954
 955         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 956                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
 957                 struct pp2_ppio_inq_statistics rx_stats;
 958
 959                 if (!rxq)
 960                         continue;
 961
 962                 idx = rxq->queue_id;
 963                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
 964                         RTE_LOG(ERR, PMD,
 965                                 "rx queue %d stats out of range (0 - %d)\n",
 966                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
 967                         continue;
 968                 }
 969
 970                 ret = pp2_ppio_inq_get_statistics(priv->ppio,
 971                                                   priv->rxq_map[idx].tc,
 972                                                   priv->rxq_map[idx].inq,
 973                                                   &rx_stats, 0);
 974                 if (unlikely(ret)) {
 975                         RTE_LOG(ERR, PMD,
 976                                 "Failed to update rx queue %d stats\n", idx);
 977                         break;
 978                 }
 979
 980                 stats->q_ibytes[idx] = rxq->bytes_recv;
 981                 stats->q_ipackets[idx] = rx_stats.enq_desc - rxq->drop_mac;
 982                 stats->q_errors[idx] = rx_stats.drop_early +
 983                                        rx_stats.drop_fullq +
 984                                        rx_stats.drop_bm +
 985                                        rxq->drop_mac;
 986                 stats->ibytes += rxq->bytes_recv;
 987                 drop_mac += rxq->drop_mac;
 988         }
 989
 990         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 991                 struct mrvl_txq *txq = dev->data->tx_queues[i];
 992                 struct pp2_ppio_outq_statistics tx_stats;
 993
 994                 if (!txq)
 995                         continue;
 996
 997                 idx = txq->queue_id;
 998                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
 999                         RTE_LOG(ERR, PMD,
1000                                 "tx queue %d stats out of range (0 - %d)\n",
1001                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
1002                 }
1003
1004                 ret = pp2_ppio_outq_get_statistics(priv->ppio, idx,
1005                                                    &tx_stats, 0);
1006                 if (unlikely(ret)) {
1007                         RTE_LOG(ERR, PMD,
1008                                 "Failed to update tx queue %d stats\n", idx);
1009                         break;
1010                 }
1011
1012                 stats->q_opackets[idx] = tx_stats.deq_desc;
1013                 stats->q_obytes[idx] = txq->bytes_sent;
1014                 stats->obytes += txq->bytes_sent;
1015         }
1016
1017         ret = pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
1018         if (unlikely(ret)) {
1019                 RTE_LOG(ERR, PMD, "Failed to update port statistics\n");
1020                 return ret;
1021         }
1022
1023         stats->ipackets += ppio_stats.rx_packets - drop_mac;
1024         stats->opackets += ppio_stats.tx_packets;
1025         stats->imissed += ppio_stats.rx_fullq_dropped +
1026                           ppio_stats.rx_bm_dropped +
1027                           ppio_stats.rx_early_dropped +
1028                           ppio_stats.rx_fifo_dropped +
1029                           ppio_stats.rx_cls_dropped;
1030         stats->ierrors = drop_mac;
1031
1032         return 0;
1033 }
1034
1035 /**
1036  * DPDK callback to clear device statistics.
1037  *
1038  * @param dev
1039  *   Pointer to Ethernet device structure.
1040  */
1041 static void
1042 mrvl_stats_reset(struct rte_eth_dev *dev)
1043 {
1044         struct mrvl_priv *priv = dev->data->dev_private;
1045         int i;
1046
1047         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1048                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
1049
1050                 pp2_ppio_inq_get_statistics(priv->ppio, priv->rxq_map[i].tc,
1051                                             priv->rxq_map[i].inq, NULL, 1);
1052                 rxq->bytes_recv = 0;
1053                 rxq->drop_mac = 0;
1054         }
1055
1056         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1057                 struct mrvl_txq *txq = dev->data->tx_queues[i];
1058
1059                 pp2_ppio_outq_get_statistics(priv->ppio, i, NULL, 1);
1060                 txq->bytes_sent = 0;
1061         }
1062
1063         pp2_ppio_get_statistics(priv->ppio, NULL, 1);
1064 }
1065
1066 /**
1067  * DPDK callback to get information about the device.
1068  *
1069  * @param dev
1070  *   Pointer to Ethernet device structure (unused).
1071  * @param info
1072  *   Info structure output buffer.
1073  */
1074 static void
1075 mrvl_dev_infos_get(struct rte_eth_dev *dev __rte_unused,
1076                    struct rte_eth_dev_info *info)
1077 {
1078         info->speed_capa = ETH_LINK_SPEED_10M |
1079                            ETH_LINK_SPEED_100M |
1080                            ETH_LINK_SPEED_1G |
1081                            ETH_LINK_SPEED_10G;
1082
1083         info->max_rx_queues = MRVL_PP2_RXQ_MAX;
1084         info->max_tx_queues = MRVL_PP2_TXQ_MAX;
1085         info->max_mac_addrs = MRVL_MAC_ADDRS_MAX;
1086
1087         info->rx_desc_lim.nb_max = MRVL_PP2_RXD_MAX;
1088         info->rx_desc_lim.nb_min = MRVL_PP2_RXD_MIN;
1089         info->rx_desc_lim.nb_align = MRVL_PP2_RXD_ALIGN;
1090
1091         info->tx_desc_lim.nb_max = MRVL_PP2_TXD_MAX;
1092         info->tx_desc_lim.nb_min = MRVL_PP2_TXD_MIN;
1093         info->tx_desc_lim.nb_align = MRVL_PP2_TXD_ALIGN;
1094
1095         info->rx_offload_capa = DEV_RX_OFFLOAD_JUMBO_FRAME |
1096                                 DEV_RX_OFFLOAD_VLAN_FILTER |
1097                                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1098                                 DEV_RX_OFFLOAD_UDP_CKSUM |
1099                                 DEV_RX_OFFLOAD_TCP_CKSUM;
1100
1101         info->tx_offload_capa = DEV_TX_OFFLOAD_IPV4_CKSUM |
1102                                 DEV_TX_OFFLOAD_UDP_CKSUM |
1103                                 DEV_TX_OFFLOAD_TCP_CKSUM;
1104
1105         info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1106                                        ETH_RSS_NONFRAG_IPV4_TCP |
1107                                        ETH_RSS_NONFRAG_IPV4_UDP;
1108
1109         /* By default packets are dropped if no descriptors are available */
1110         info->default_rxconf.rx_drop_en = 1;
1111
1112         info->max_rx_pktlen = MRVL_PKT_SIZE_MAX;
1113 }
1114
1115 /**
1116  * Return supported packet types.
1117  *
1118  * @param dev
1119  *   Pointer to Ethernet device structure (unused).
1120  *
1121  * @return
1122  *   Const pointer to the table with supported packet types.
1123  */
1124 static const uint32_t *
1125 mrvl_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1126 {
1127         static const uint32_t ptypes[] = {
1128                 RTE_PTYPE_L2_ETHER,
1129                 RTE_PTYPE_L3_IPV4,
1130                 RTE_PTYPE_L3_IPV4_EXT,
1131                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
1132                 RTE_PTYPE_L3_IPV6,
1133                 RTE_PTYPE_L3_IPV6_EXT,
1134                 RTE_PTYPE_L2_ETHER_ARP,
1135                 RTE_PTYPE_L4_TCP,
1136                 RTE_PTYPE_L4_UDP
1137         };
1138
1139         return ptypes;
1140 }
1141
1142 /**
1143  * DPDK callback to get information about specific receive queue.
1144  *
1145  * @param dev
1146  *   Pointer to Ethernet device structure.
1147  * @param rx_queue_id
1148  *   Receive queue index.
1149  * @param qinfo
1150  *   Receive queue information structure.
1151  */
1152 static void mrvl_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
1153                               struct rte_eth_rxq_info *qinfo)
1154 {
1155         struct mrvl_rxq *q = dev->data->rx_queues[rx_queue_id];
1156         struct mrvl_priv *priv = dev->data->dev_private;
1157         int inq = priv->rxq_map[rx_queue_id].inq;
1158         int tc = priv->rxq_map[rx_queue_id].tc;
1159         struct pp2_ppio_tc_params *tc_params =
1160                 &priv->ppio_params.inqs_params.tcs_params[tc];
1161
1162         qinfo->mp = q->mp;
1163         qinfo->nb_desc = tc_params->inqs_params[inq].size;
1164 }
1165
1166 /**
1167  * DPDK callback to get information about specific transmit queue.
1168  *
1169  * @param dev
1170  *   Pointer to Ethernet device structure.
1171  * @param tx_queue_id
1172  *   Transmit queue index.
1173  * @param qinfo
1174  *   Transmit queue information structure.
1175  */
1176 static void mrvl_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
1177                               struct rte_eth_txq_info *qinfo)
1178 {
1179         struct mrvl_priv *priv = dev->data->dev_private;
1180
1181         qinfo->nb_desc =
1182                 priv->ppio_params.outqs_params.outqs_params[tx_queue_id].size;
1183 }
1184
1185 /**
1186  * DPDK callback to Configure a VLAN filter.
1187  *
1188  * @param dev
1189  *   Pointer to Ethernet device structure.
1190  * @param vlan_id
1191  *   VLAN ID to filter.
1192  * @param on
1193  *   Toggle filter.
1194  *
1195  * @return
1196  *   0 on success, negative error value otherwise.
1197  */
1198 static int
1199 mrvl_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1200 {
1201         struct mrvl_priv *priv = dev->data->dev_private;
1202
1203         return on ? pp2_ppio_add_vlan(priv->ppio, vlan_id) :
1204                     pp2_ppio_remove_vlan(priv->ppio, vlan_id);
1205 }
1206
1207 /**
1208  * Release buffers to hardware bpool (buffer-pool)
1209  *
1210  * @param rxq
1211  *   Receive queue pointer.
1212  * @param num
1213  *   Number of buffers to release to bpool.
1214  *
1215  * @return
1216  *   0 on success, negative error value otherwise.
1217  */
1218 static int
1219 mrvl_fill_bpool(struct mrvl_rxq *rxq, int num)
1220 {
1221         struct buff_release_entry entries[MRVL_PP2_RXD_MAX];
1222         struct rte_mbuf *mbufs[MRVL_PP2_RXD_MAX];
1223         int i, ret;
1224         unsigned int core_id;
1225         struct pp2_hif *hif;
1226         struct pp2_bpool *bpool;
1227
1228         core_id = rte_lcore_id();
1229         if (core_id == LCORE_ID_ANY)
1230                 core_id = 0;
1231
1232         hif = mrvl_get_hif(rxq->priv, core_id);
1233         if (!hif)
1234                 return -1;
1235
1236         bpool = rxq->priv->bpool;
1237
1238         ret = rte_pktmbuf_alloc_bulk(rxq->mp, mbufs, num);
1239         if (ret)
1240                 return ret;
1241
1242         if (cookie_addr_high == MRVL_COOKIE_ADDR_INVALID)
1243                 cookie_addr_high =
1244                         (uint64_t)mbufs[0] & MRVL_COOKIE_HIGH_ADDR_MASK;
1245
1246         for (i = 0; i < num; i++) {
1247                 if (((uint64_t)mbufs[i] & MRVL_COOKIE_HIGH_ADDR_MASK)
1248                         != cookie_addr_high) {
1249                         RTE_LOG(ERR, PMD,
1250                                 "mbuf virtual addr high 0x%lx out of range\n",
1251                                 (uint64_t)mbufs[i] >> 32);
1252                         goto out;
1253                 }
1254
1255                 entries[i].buff.addr =
1256                         rte_mbuf_data_iova_default(mbufs[i]);
1257                 entries[i].buff.cookie = (pp2_cookie_t)(uint64_t)mbufs[i];
1258                 entries[i].bpool = bpool;
1259         }
1260
1261         pp2_bpool_put_buffs(hif, entries, (uint16_t *)&i);
1262         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] += i;
1263
1264         if (i != num)
1265                 goto out;
1266
1267         return 0;
1268 out:
1269         for (; i < num; i++)
1270                 rte_pktmbuf_free(mbufs[i]);
1271
1272         return -1;
1273 }
1274
1275 /**
1276  * DPDK callback to configure the receive queue.
1277  *
1278  * @param dev
1279  *   Pointer to Ethernet device structure.
1280  * @param idx
1281  *   RX queue index.
1282  * @param desc
1283  *   Number of descriptors to configure in queue.
1284  * @param socket
1285  *   NUMA socket on which memory must be allocated.
1286  * @param conf
1287  *   Thresholds parameters (unused_).
1288  * @param mp
1289  *   Memory pool for buffer allocations.
1290  *
1291  * @return
1292  *   0 on success, negative error value otherwise.
1293  */
1294 static int
1295 mrvl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1296                     unsigned int socket,
1297                     const struct rte_eth_rxconf *conf __rte_unused,
1298                     struct rte_mempool *mp)
1299 {
1300         struct mrvl_priv *priv = dev->data->dev_private;
1301         struct mrvl_rxq *rxq;
1302         uint32_t min_size,
1303                  max_rx_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
1304         int ret, tc, inq;
1305
1306         if (priv->rxq_map[idx].tc == MRVL_UNKNOWN_TC) {
1307                 /*
1308                  * Unknown TC mapping, mapping will not have a correct queue.
1309                  */
1310                 RTE_LOG(ERR, PMD, "Unknown TC mapping for queue %hu eth%hhu\n",
1311                         idx, priv->ppio_id);
1312                 return -EFAULT;
1313         }
1314
1315         min_size = rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM -
1316                    MRVL_PKT_EFFEC_OFFS;
1317         if (min_size < max_rx_pkt_len) {
1318                 RTE_LOG(ERR, PMD,
1319                         "Mbuf size must be increased to %u bytes to hold up to %u bytes of data.\n",
1320                         max_rx_pkt_len + RTE_PKTMBUF_HEADROOM +
1321                         MRVL_PKT_EFFEC_OFFS,
1322                         max_rx_pkt_len);
1323                 return -EINVAL;
1324         }
1325
1326         if (dev->data->rx_queues[idx]) {
1327                 rte_free(dev->data->rx_queues[idx]);
1328                 dev->data->rx_queues[idx] = NULL;
1329         }
1330
1331         rxq = rte_zmalloc_socket("rxq", sizeof(*rxq), 0, socket);
1332         if (!rxq)
1333                 return -ENOMEM;
1334
1335         rxq->priv = priv;
1336         rxq->mp = mp;
1337         rxq->cksum_enabled = dev->data->dev_conf.rxmode.hw_ip_checksum;
1338         rxq->queue_id = idx;
1339         rxq->port_id = dev->data->port_id;
1340         mrvl_port_to_bpool_lookup[rxq->port_id] = priv->bpool;
1341
1342         tc = priv->rxq_map[rxq->queue_id].tc,
1343         inq = priv->rxq_map[rxq->queue_id].inq;
1344         priv->ppio_params.inqs_params.tcs_params[tc].inqs_params[inq].size =
1345                 desc;
1346
1347         ret = mrvl_fill_bpool(rxq, desc);
1348         if (ret) {
1349                 rte_free(rxq);
1350                 return ret;
1351         }
1352
1353         priv->bpool_init_size += desc;
1354
1355         dev->data->rx_queues[idx] = rxq;
1356
1357         return 0;
1358 }
1359
1360 /**
1361  * DPDK callback to release the receive queue.
1362  *
1363  * @param rxq
1364  *   Generic receive queue pointer.
1365  */
1366 static void
1367 mrvl_rx_queue_release(void *rxq)
1368 {
1369         struct mrvl_rxq *q = rxq;
1370         struct pp2_ppio_tc_params *tc_params;
1371         int i, num, tc, inq;
1372         struct pp2_hif *hif;
1373         unsigned int core_id = rte_lcore_id();
1374
1375         if (core_id == LCORE_ID_ANY)
1376                 core_id = 0;
1377
1378         if (!q)
1379                 return;
1380
1381         hif = mrvl_get_hif(q->priv, core_id);
1382
1383         if (!hif)
1384                 return;
1385
1386         tc = q->priv->rxq_map[q->queue_id].tc;
1387         inq = q->priv->rxq_map[q->queue_id].inq;
1388         tc_params = &q->priv->ppio_params.inqs_params.tcs_params[tc];
1389         num = tc_params->inqs_params[inq].size;
1390         for (i = 0; i < num; i++) {
1391                 struct pp2_buff_inf inf;
1392                 uint64_t addr;
1393
1394                 pp2_bpool_get_buff(hif, q->priv->bpool, &inf);
1395                 addr = cookie_addr_high | inf.cookie;
1396                 rte_pktmbuf_free((struct rte_mbuf *)addr);
1397         }
1398
1399         rte_free(q);
1400 }
1401
1402 /**
1403  * DPDK callback to configure the transmit queue.
1404  *
1405  * @param dev
1406  *   Pointer to Ethernet device structure.
1407  * @param idx
1408  *   Transmit queue index.
1409  * @param desc
1410  *   Number of descriptors to configure in the queue.
1411  * @param socket
1412  *   NUMA socket on which memory must be allocated.
1413  * @param conf
1414  *   Thresholds parameters (unused).
1415  *
1416  * @return
1417  *   0 on success, negative error value otherwise.
1418  */
1419 static int
1420 mrvl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1421                     unsigned int socket,
1422                     const struct rte_eth_txconf *conf __rte_unused)
1423 {
1424         struct mrvl_priv *priv = dev->data->dev_private;
1425         struct mrvl_txq *txq;
1426
1427         if (dev->data->tx_queues[idx]) {
1428                 rte_free(dev->data->tx_queues[idx]);
1429                 dev->data->tx_queues[idx] = NULL;
1430         }
1431
1432         txq = rte_zmalloc_socket("txq", sizeof(*txq), 0, socket);
1433         if (!txq)
1434                 return -ENOMEM;
1435
1436         txq->priv = priv;
1437         txq->queue_id = idx;
1438         txq->port_id = dev->data->port_id;
1439         dev->data->tx_queues[idx] = txq;
1440
1441         priv->ppio_params.outqs_params.outqs_params[idx].size = desc;
1442         priv->ppio_params.outqs_params.outqs_params[idx].weight = 1;
1443
1444         return 0;
1445 }
1446
1447 /**
1448  * DPDK callback to release the transmit queue.
1449  *
1450  * @param txq
1451  *   Generic transmit queue pointer.
1452  */
1453 static void
1454 mrvl_tx_queue_release(void *txq)
1455 {
1456         struct mrvl_txq *q = txq;
1457
1458         if (!q)
1459                 return;
1460
1461         rte_free(q);
1462 }
1463
1464 /**
1465  * Update RSS hash configuration
1466  *
1467  * @param dev
1468  *   Pointer to Ethernet device structure.
1469  * @param rss_conf
1470  *   Pointer to RSS configuration.
1471  *
1472  * @return
1473  *   0 on success, negative error value otherwise.
1474  */
1475 static int
1476 mrvl_rss_hash_update(struct rte_eth_dev *dev,
1477                      struct rte_eth_rss_conf *rss_conf)
1478 {
1479         struct mrvl_priv *priv = dev->data->dev_private;
1480
1481         return mrvl_configure_rss(priv, rss_conf);
1482 }
1483
1484 /**
1485  * DPDK callback to get RSS hash configuration.
1486  *
1487  * @param dev
1488  *   Pointer to Ethernet device structure.
1489  * @rss_conf
1490  *   Pointer to RSS configuration.
1491  *
1492  * @return
1493  *   Always 0.
1494  */
1495 static int
1496 mrvl_rss_hash_conf_get(struct rte_eth_dev *dev,
1497                        struct rte_eth_rss_conf *rss_conf)
1498 {
1499         struct mrvl_priv *priv = dev->data->dev_private;
1500         enum pp2_ppio_hash_type hash_type =
1501                 priv->ppio_params.inqs_params.hash_type;
1502
1503         rss_conf->rss_key = NULL;
1504
1505         if (hash_type == PP2_PPIO_HASH_T_NONE)
1506                 rss_conf->rss_hf = 0;
1507         else if (hash_type == PP2_PPIO_HASH_T_2_TUPLE)
1508                 rss_conf->rss_hf = ETH_RSS_IPV4;
1509         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && priv->rss_hf_tcp)
1510                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_TCP;
1511         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && !priv->rss_hf_tcp)
1512                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_UDP;
1513
1514         return 0;
1515 }
1516
1517 static const struct eth_dev_ops mrvl_ops = {
1518         .dev_configure = mrvl_dev_configure,
1519         .dev_start = mrvl_dev_start,
1520         .dev_stop = mrvl_dev_stop,
1521         .dev_set_link_up = mrvl_dev_set_link_up,
1522         .dev_set_link_down = mrvl_dev_set_link_down,
1523         .dev_close = mrvl_dev_close,
1524         .link_update = mrvl_link_update,
1525         .promiscuous_enable = mrvl_promiscuous_enable,
1526         .allmulticast_enable = mrvl_allmulticast_enable,
1527         .promiscuous_disable = mrvl_promiscuous_disable,
1528         .allmulticast_disable = mrvl_allmulticast_disable,
1529         .mac_addr_remove = mrvl_mac_addr_remove,
1530         .mac_addr_add = mrvl_mac_addr_add,
1531         .mac_addr_set = mrvl_mac_addr_set,
1532         .mtu_set = mrvl_mtu_set,
1533         .stats_get = mrvl_stats_get,
1534         .stats_reset = mrvl_stats_reset,
1535         .dev_infos_get = mrvl_dev_infos_get,
1536         .dev_supported_ptypes_get = mrvl_dev_supported_ptypes_get,
1537         .rxq_info_get = mrvl_rxq_info_get,
1538         .txq_info_get = mrvl_txq_info_get,
1539         .vlan_filter_set = mrvl_vlan_filter_set,
1540         .rx_queue_setup = mrvl_rx_queue_setup,
1541         .rx_queue_release = mrvl_rx_queue_release,
1542         .tx_queue_setup = mrvl_tx_queue_setup,
1543         .tx_queue_release = mrvl_tx_queue_release,
1544         .rss_hash_update = mrvl_rss_hash_update,
1545         .rss_hash_conf_get = mrvl_rss_hash_conf_get,
1546 };
1547
1548 /**
1549  * Return packet type information and l3/l4 offsets.
1550  *
1551  * @param desc
1552  *   Pointer to the received packet descriptor.
1553  * @param l3_offset
1554  *   l3 packet offset.
1555  * @param l4_offset
1556  *   l4 packet offset.
1557  *
1558  * @return
1559  *   Packet type information.
1560  */
1561 static inline uint64_t
1562 mrvl_desc_to_packet_type_and_offset(struct pp2_ppio_desc *desc,
1563                                     uint8_t *l3_offset, uint8_t *l4_offset)
1564 {
1565         enum pp2_inq_l3_type l3_type;
1566         enum pp2_inq_l4_type l4_type;
1567         uint64_t packet_type;
1568
1569         pp2_ppio_inq_desc_get_l3_info(desc, &l3_type, l3_offset);
1570         pp2_ppio_inq_desc_get_l4_info(desc, &l4_type, l4_offset);
1571
1572         packet_type = RTE_PTYPE_L2_ETHER;
1573
1574         switch (l3_type) {
1575         case PP2_INQ_L3_TYPE_IPV4_NO_OPTS:
1576                 packet_type |= RTE_PTYPE_L3_IPV4;
1577                 break;
1578         case PP2_INQ_L3_TYPE_IPV4_OK:
1579                 packet_type |= RTE_PTYPE_L3_IPV4_EXT;
1580                 break;
1581         case PP2_INQ_L3_TYPE_IPV4_TTL_ZERO:
1582                 packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
1583                 break;
1584         case PP2_INQ_L3_TYPE_IPV6_NO_EXT:
1585                 packet_type |= RTE_PTYPE_L3_IPV6;
1586                 break;
1587         case PP2_INQ_L3_TYPE_IPV6_EXT:
1588                 packet_type |= RTE_PTYPE_L3_IPV6_EXT;
1589                 break;
1590         case PP2_INQ_L3_TYPE_ARP:
1591                 packet_type |= RTE_PTYPE_L2_ETHER_ARP;
1592                 /*
1593                  * In case of ARP l4_offset is set to wrong value.
1594                  * Set it to proper one so that later on mbuf->l3_len can be
1595                  * calculated subtracting l4_offset and l3_offset.
1596                  */
1597                 *l4_offset = *l3_offset + MRVL_ARP_LENGTH;
1598                 break;
1599         default:
1600                 RTE_LOG(DEBUG, PMD, "Failed to recognise l3 packet type\n");
1601                 break;
1602         }
1603
1604         switch (l4_type) {
1605         case PP2_INQ_L4_TYPE_TCP:
1606                 packet_type |= RTE_PTYPE_L4_TCP;
1607                 break;
1608         case PP2_INQ_L4_TYPE_UDP:
1609                 packet_type |= RTE_PTYPE_L4_UDP;
1610                 break;
1611         default:
1612                 RTE_LOG(DEBUG, PMD, "Failed to recognise l4 packet type\n");
1613                 break;
1614         }
1615
1616         return packet_type;
1617 }
1618
1619 /**
1620  * Get offload information from the received packet descriptor.
1621  *
1622  * @param desc
1623  *   Pointer to the received packet descriptor.
1624  *
1625  * @return
1626  *   Mbuf offload flags.
1627  */
1628 static inline uint64_t
1629 mrvl_desc_to_ol_flags(struct pp2_ppio_desc *desc)
1630 {
1631         uint64_t flags;
1632         enum pp2_inq_desc_status status;
1633
1634         status = pp2_ppio_inq_desc_get_l3_pkt_error(desc);
1635         if (unlikely(status != PP2_DESC_ERR_OK))
1636                 flags = PKT_RX_IP_CKSUM_BAD;
1637         else
1638                 flags = PKT_RX_IP_CKSUM_GOOD;
1639
1640         status = pp2_ppio_inq_desc_get_l4_pkt_error(desc);
1641         if (unlikely(status != PP2_DESC_ERR_OK))
1642                 flags |= PKT_RX_L4_CKSUM_BAD;
1643         else
1644                 flags |= PKT_RX_L4_CKSUM_GOOD;
1645
1646         return flags;
1647 }
1648
1649 /**
1650  * DPDK callback for receive.
1651  *
1652  * @param rxq
1653  *   Generic pointer to the receive queue.
1654  * @param rx_pkts
1655  *   Array to store received packets.
1656  * @param nb_pkts
1657  *   Maximum number of packets in array.
1658  *
1659  * @return
1660  *   Number of packets successfully received.
1661  */
1662 static uint16_t
1663 mrvl_rx_pkt_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
1664 {
1665         struct mrvl_rxq *q = rxq;
1666         struct pp2_ppio_desc descs[nb_pkts];
1667         struct pp2_bpool *bpool;
1668         int i, ret, rx_done = 0;
1669         int num;
1670         struct pp2_hif *hif;
1671         unsigned int core_id = rte_lcore_id();
1672
1673         hif = mrvl_get_hif(q->priv, core_id);
1674
1675         if (unlikely(!q->priv->ppio || !hif))
1676                 return 0;
1677
1678         bpool = q->priv->bpool;
1679
1680         ret = pp2_ppio_recv(q->priv->ppio, q->priv->rxq_map[q->queue_id].tc,
1681                             q->priv->rxq_map[q->queue_id].inq, descs, &nb_pkts);
1682         if (unlikely(ret < 0)) {
1683                 RTE_LOG(ERR, PMD, "Failed to receive packets\n");
1684                 return 0;
1685         }
1686         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] -= nb_pkts;
1687
1688         for (i = 0; i < nb_pkts; i++) {
1689                 struct rte_mbuf *mbuf;
1690                 uint8_t l3_offset, l4_offset;
1691                 enum pp2_inq_desc_status status;
1692                 uint64_t addr;
1693
1694                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
1695                         struct pp2_ppio_desc *pref_desc;
1696                         u64 pref_addr;
1697
1698                         pref_desc = &descs[i + MRVL_MUSDK_PREFETCH_SHIFT];
1699                         pref_addr = cookie_addr_high |
1700                                     pp2_ppio_inq_desc_get_cookie(pref_desc);
1701                         rte_mbuf_prefetch_part1((struct rte_mbuf *)(pref_addr));
1702                         rte_mbuf_prefetch_part2((struct rte_mbuf *)(pref_addr));
1703                 }
1704
1705                 addr = cookie_addr_high |
1706                        pp2_ppio_inq_desc_get_cookie(&descs[i]);
1707                 mbuf = (struct rte_mbuf *)addr;
1708                 rte_pktmbuf_reset(mbuf);
1709
1710                 /* drop packet in case of mac, overrun or resource error */
1711                 status = pp2_ppio_inq_desc_get_l2_pkt_error(&descs[i]);
1712                 if (unlikely(status != PP2_DESC_ERR_OK)) {
1713                         struct pp2_buff_inf binf = {
1714                                 .addr = rte_mbuf_data_iova_default(mbuf),
1715                                 .cookie = (pp2_cookie_t)(uint64_t)mbuf,
1716                         };
1717
1718                         pp2_bpool_put_buff(hif, bpool, &binf);
1719                         mrvl_port_bpool_size
1720                                 [bpool->pp2_id][bpool->id][core_id]++;
1721                         q->drop_mac++;
1722                         continue;
1723                 }
1724
1725                 mbuf->data_off += MRVL_PKT_EFFEC_OFFS;
1726                 mbuf->pkt_len = pp2_ppio_inq_desc_get_pkt_len(&descs[i]);
1727                 mbuf->data_len = mbuf->pkt_len;
1728                 mbuf->port = q->port_id;
1729                 mbuf->packet_type =
1730                         mrvl_desc_to_packet_type_and_offset(&descs[i],
1731                                                             &l3_offset,
1732                                                             &l4_offset);
1733                 mbuf->l2_len = l3_offset;
1734                 mbuf->l3_len = l4_offset - l3_offset;
1735
1736                 if (likely(q->cksum_enabled))
1737                         mbuf->ol_flags = mrvl_desc_to_ol_flags(&descs[i]);
1738
1739                 rx_pkts[rx_done++] = mbuf;
1740                 q->bytes_recv += mbuf->pkt_len;
1741         }
1742
1743         if (rte_spinlock_trylock(&q->priv->lock) == 1) {
1744                 num = mrvl_get_bpool_size(bpool->pp2_id, bpool->id);
1745
1746                 if (unlikely(num <= q->priv->bpool_min_size ||
1747                              (!rx_done && num < q->priv->bpool_init_size))) {
1748                         ret = mrvl_fill_bpool(q, MRVL_BURST_SIZE);
1749                         if (ret)
1750                                 RTE_LOG(ERR, PMD, "Failed to fill bpool\n");
1751                 } else if (unlikely(num > q->priv->bpool_max_size)) {
1752                         int i;
1753                         int pkt_to_remove = num - q->priv->bpool_init_size;
1754                         struct rte_mbuf *mbuf;
1755                         struct pp2_buff_inf buff;
1756
1757                         RTE_LOG(DEBUG, PMD,
1758                                 "\nport-%d:%d: bpool %d oversize - remove %d buffers (pool size: %d -> %d)\n",
1759                                 bpool->pp2_id, q->priv->ppio->port_id,
1760                                 bpool->id, pkt_to_remove, num,
1761                                 q->priv->bpool_init_size);
1762
1763                         for (i = 0; i < pkt_to_remove; i++) {
1764                                 ret = pp2_bpool_get_buff(hif, bpool, &buff);
1765                                 if (ret)
1766                                         break;
1767                                 mbuf = (struct rte_mbuf *)
1768                                         (cookie_addr_high | buff.cookie);
1769                                 rte_pktmbuf_free(mbuf);
1770                         }
1771                         mrvl_port_bpool_size
1772                                 [bpool->pp2_id][bpool->id][core_id] -= i;
1773                 }
1774                 rte_spinlock_unlock(&q->priv->lock);
1775         }
1776
1777         return rx_done;
1778 }
1779
1780 /**
1781  * Prepare offload information.
1782  *
1783  * @param ol_flags
1784  *   Offload flags.
1785  * @param packet_type
1786  *   Packet type bitfield.
1787  * @param l3_type
1788  *   Pointer to the pp2_ouq_l3_type structure.
1789  * @param l4_type
1790  *   Pointer to the pp2_outq_l4_type structure.
1791  * @param gen_l3_cksum
1792  *   Will be set to 1 in case l3 checksum is computed.
1793  * @param l4_cksum
1794  *   Will be set to 1 in case l4 checksum is computed.
1795  *
1796  * @return
1797  *   0 on success, negative error value otherwise.
1798  */
1799 static inline int
1800 mrvl_prepare_proto_info(uint64_t ol_flags, uint32_t packet_type,
1801                         enum pp2_outq_l3_type *l3_type,
1802                         enum pp2_outq_l4_type *l4_type,
1803                         int *gen_l3_cksum,
1804                         int *gen_l4_cksum)
1805 {
1806         /*
1807          * Based on ol_flags prepare information
1808          * for pp2_ppio_outq_desc_set_proto_info() which setups descriptor
1809          * for offloading.
1810          */
1811         if (ol_flags & PKT_TX_IPV4) {
1812                 *l3_type = PP2_OUTQ_L3_TYPE_IPV4;
1813                 *gen_l3_cksum = ol_flags & PKT_TX_IP_CKSUM ? 1 : 0;
1814         } else if (ol_flags & PKT_TX_IPV6) {
1815                 *l3_type = PP2_OUTQ_L3_TYPE_IPV6;
1816                 /* no checksum for ipv6 header */
1817                 *gen_l3_cksum = 0;
1818         } else {
1819                 /* if something different then stop processing */
1820                 return -1;
1821         }
1822
1823         ol_flags &= PKT_TX_L4_MASK;
1824         if ((packet_type & RTE_PTYPE_L4_TCP) &&
1825             ol_flags == PKT_TX_TCP_CKSUM) {
1826                 *l4_type = PP2_OUTQ_L4_TYPE_TCP;
1827                 *gen_l4_cksum = 1;
1828         } else if ((packet_type & RTE_PTYPE_L4_UDP) &&
1829                    ol_flags == PKT_TX_UDP_CKSUM) {
1830                 *l4_type = PP2_OUTQ_L4_TYPE_UDP;
1831                 *gen_l4_cksum = 1;
1832         } else {
1833                 *l4_type = PP2_OUTQ_L4_TYPE_OTHER;
1834                 /* no checksum for other type */
1835                 *gen_l4_cksum = 0;
1836         }
1837
1838         return 0;
1839 }
1840
1841 /**
1842  * Release already sent buffers to bpool (buffer-pool).
1843  *
1844  * @param ppio
1845  *   Pointer to the port structure.
1846  * @param hif
1847  *   Pointer to the MUSDK hardware interface.
1848  * @param sq
1849  *   Pointer to the shadow queue.
1850  * @param qid
1851  *   Queue id number.
1852  * @param force
1853  *   Force releasing packets.
1854  */
1855 static inline void
1856 mrvl_free_sent_buffers(struct pp2_ppio *ppio, struct pp2_hif *hif,
1857                        unsigned int core_id, struct mrvl_shadow_txq *sq,
1858                        int qid, int force)
1859 {
1860         struct buff_release_entry *entry;
1861         uint16_t nb_done = 0, num = 0, skip_bufs = 0;
1862         int i;
1863
1864         pp2_ppio_get_num_outq_done(ppio, hif, qid, &nb_done);
1865
1866         sq->num_to_release += nb_done;
1867
1868         if (likely(!force &&
1869                    sq->num_to_release < MRVL_PP2_BUF_RELEASE_BURST_SIZE))
1870                 return;
1871
1872         nb_done = sq->num_to_release;
1873         sq->num_to_release = 0;
1874
1875         for (i = 0; i < nb_done; i++) {
1876                 entry = &sq->ent[sq->tail + num];
1877                 if (unlikely(!entry->buff.addr)) {
1878                         RTE_LOG(ERR, PMD,
1879                                 "Shadow memory @%d: cookie(%lx), pa(%lx)!\n",
1880                                 sq->tail, (u64)entry->buff.cookie,
1881                                 (u64)entry->buff.addr);
1882                         skip_bufs = 1;
1883                         goto skip;
1884                 }
1885
1886                 if (unlikely(!entry->bpool)) {
1887                         struct rte_mbuf *mbuf;
1888
1889                         mbuf = (struct rte_mbuf *)
1890                                (cookie_addr_high | entry->buff.cookie);
1891                         rte_pktmbuf_free(mbuf);
1892                         skip_bufs = 1;
1893                         goto skip;
1894                 }
1895
1896                 mrvl_port_bpool_size
1897                         [entry->bpool->pp2_id][entry->bpool->id][core_id]++;
1898                 num++;
1899                 if (unlikely(sq->tail + num == MRVL_PP2_TX_SHADOWQ_SIZE))
1900                         goto skip;
1901                 continue;
1902 skip:
1903                 if (likely(num))
1904                         pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
1905                 num += skip_bufs;
1906                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
1907                 sq->size -= num;
1908                 num = 0;
1909                 skip_bufs = 0;
1910         }
1911
1912         if (likely(num)) {
1913                 pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
1914                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
1915                 sq->size -= num;
1916         }
1917 }
1918
1919 /**
1920  * DPDK callback for transmit.
1921  *
1922  * @param txq
1923  *   Generic pointer transmit queue.
1924  * @param tx_pkts
1925  *   Packets to transmit.
1926  * @param nb_pkts
1927  *   Number of packets in array.
1928  *
1929  * @return
1930  *   Number of packets successfully transmitted.
1931  */
1932 static uint16_t
1933 mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
1934 {
1935         struct mrvl_txq *q = txq;
1936         struct mrvl_shadow_txq *sq;
1937         struct pp2_hif *hif;
1938         struct pp2_ppio_desc descs[nb_pkts];
1939         unsigned int core_id = rte_lcore_id();
1940         int i, ret, bytes_sent = 0;
1941         uint16_t num, sq_free_size;
1942         uint64_t addr;
1943
1944         hif = mrvl_get_hif(q->priv, core_id);
1945         sq = &q->shadow_txqs[core_id];
1946
1947         if (unlikely(!q->priv->ppio || !hif))
1948                 return 0;
1949
1950         if (sq->size)
1951                 mrvl_free_sent_buffers(q->priv->ppio, hif, core_id,
1952                                        sq, q->queue_id, 0);
1953
1954         sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1;
1955         if (unlikely(nb_pkts > sq_free_size)) {
1956                 RTE_LOG(DEBUG, PMD,
1957                         "No room in shadow queue for %d packets! %d packets will be sent.\n",
1958                         nb_pkts, sq_free_size);
1959                 nb_pkts = sq_free_size;
1960         }
1961
1962         for (i = 0; i < nb_pkts; i++) {
1963                 struct rte_mbuf *mbuf = tx_pkts[i];
1964                 int gen_l3_cksum, gen_l4_cksum;
1965                 enum pp2_outq_l3_type l3_type;
1966                 enum pp2_outq_l4_type l4_type;
1967
1968                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
1969                         struct rte_mbuf *pref_pkt_hdr;
1970
1971                         pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT];
1972                         rte_mbuf_prefetch_part1(pref_pkt_hdr);
1973                         rte_mbuf_prefetch_part2(pref_pkt_hdr);
1974                 }
1975
1976                 sq->ent[sq->head].buff.cookie = (pp2_cookie_t)(uint64_t)mbuf;
1977                 sq->ent[sq->head].buff.addr =
1978                         rte_mbuf_data_iova_default(mbuf);
1979                 sq->ent[sq->head].bpool =
1980                         (unlikely(mbuf->port == 0xff || mbuf->refcnt > 1)) ?
1981                          NULL : mrvl_port_to_bpool_lookup[mbuf->port];
1982                 sq->head = (sq->head + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
1983                 sq->size++;
1984
1985                 pp2_ppio_outq_desc_reset(&descs[i]);
1986                 pp2_ppio_outq_desc_set_phys_addr(&descs[i],
1987                                                  rte_pktmbuf_iova(mbuf));
1988                 pp2_ppio_outq_desc_set_pkt_offset(&descs[i], 0);
1989                 pp2_ppio_outq_desc_set_pkt_len(&descs[i],
1990                                                rte_pktmbuf_pkt_len(mbuf));
1991
1992                 bytes_sent += rte_pktmbuf_pkt_len(mbuf);
1993                 /*
1994                  * in case unsupported ol_flags were passed
1995                  * do not update descriptor offload information
1996                  */
1997                 ret = mrvl_prepare_proto_info(mbuf->ol_flags, mbuf->packet_type,
1998                                               &l3_type, &l4_type, &gen_l3_cksum,
1999                                               &gen_l4_cksum);
2000                 if (unlikely(ret))
2001                         continue;
2002
2003                 pp2_ppio_outq_desc_set_proto_info(&descs[i], l3_type, l4_type,
2004                                                   mbuf->l2_len,
2005                                                   mbuf->l2_len + mbuf->l3_len,
2006                                                   gen_l3_cksum, gen_l4_cksum);
2007         }
2008
2009         num = nb_pkts;
2010         pp2_ppio_send(q->priv->ppio, hif, q->queue_id, descs, &nb_pkts);
2011         /* number of packets that were not sent */
2012         if (unlikely(num > nb_pkts)) {
2013                 for (i = nb_pkts; i < num; i++) {
2014                         sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) &
2015                                 MRVL_PP2_TX_SHADOWQ_MASK;
2016                         addr = cookie_addr_high | sq->ent[sq->head].buff.cookie;
2017                         bytes_sent -=
2018                                 rte_pktmbuf_pkt_len((struct rte_mbuf *)addr);
2019                 }
2020                 sq->size -= num - nb_pkts;
2021         }
2022
2023         q->bytes_sent += bytes_sent;
2024
2025         return nb_pkts;
2026 }
2027
2028 /**
2029  * Initialize packet processor.
2030  *
2031  * @return
2032  *   0 on success, negative error value otherwise.
2033  */
2034 static int
2035 mrvl_init_pp2(void)
2036 {
2037         struct pp2_init_params init_params;
2038
2039         memset(&init_params, 0, sizeof(init_params));
2040         init_params.hif_reserved_map = MRVL_MUSDK_HIFS_RESERVED;
2041         init_params.bm_pool_reserved_map = MRVL_MUSDK_BPOOLS_RESERVED;
2042         init_params.rss_tbl_reserved_map = MRVL_MUSDK_RSS_RESERVED;
2043
2044         return pp2_init(&init_params);
2045 }
2046
2047 /**
2048  * Deinitialize packet processor.
2049  *
2050  * @return
2051  *   0 on success, negative error value otherwise.
2052  */
2053 static void
2054 mrvl_deinit_pp2(void)
2055 {
2056         pp2_deinit();
2057 }
2058
2059 /**
2060  * Create private device structure.
2061  *
2062  * @param dev_name
2063  *   Pointer to the port name passed in the initialization parameters.
2064  *
2065  * @return
2066  *   Pointer to the newly allocated private device structure.
2067  */
2068 static struct mrvl_priv *
2069 mrvl_priv_create(const char *dev_name)
2070 {
2071         struct pp2_bpool_params bpool_params;
2072         char match[MRVL_MATCH_LEN];
2073         struct mrvl_priv *priv;
2074         int ret, bpool_bit;
2075
2076         priv = rte_zmalloc_socket(dev_name, sizeof(*priv), 0, rte_socket_id());
2077         if (!priv)
2078                 return NULL;
2079
2080         ret = pp2_netdev_get_ppio_info((char *)(uintptr_t)dev_name,
2081                                        &priv->pp_id, &priv->ppio_id);
2082         if (ret)
2083                 goto out_free_priv;
2084
2085         bpool_bit = mrvl_reserve_bit(&used_bpools[priv->pp_id],
2086                                      PP2_BPOOL_NUM_POOLS);
2087         if (bpool_bit < 0)
2088                 goto out_free_priv;
2089         priv->bpool_bit = bpool_bit;
2090
2091         snprintf(match, sizeof(match), "pool-%d:%d", priv->pp_id,
2092                  priv->bpool_bit);
2093         memset(&bpool_params, 0, sizeof(bpool_params));
2094         bpool_params.match = match;
2095         bpool_params.buff_len = MRVL_PKT_SIZE_MAX + MRVL_PKT_EFFEC_OFFS;
2096         ret = pp2_bpool_init(&bpool_params, &priv->bpool);
2097         if (ret)
2098                 goto out_clear_bpool_bit;
2099
2100         priv->ppio_params.type = PP2_PPIO_T_NIC;
2101         rte_spinlock_init(&priv->lock);
2102
2103         return priv;
2104 out_clear_bpool_bit:
2105         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
2106 out_free_priv:
2107         rte_free(priv);
2108         return NULL;
2109 }
2110
2111 /**
2112  * Create device representing Ethernet port.
2113  *
2114  * @param name
2115  *   Pointer to the port's name.
2116  *
2117  * @return
2118  *   0 on success, negative error value otherwise.
2119  */
2120 static int
2121 mrvl_eth_dev_create(struct rte_vdev_device *vdev, const char *name)
2122 {
2123         int ret, fd = socket(AF_INET, SOCK_DGRAM, 0);
2124         struct rte_eth_dev *eth_dev;
2125         struct mrvl_priv *priv;
2126         struct ifreq req;
2127
2128         eth_dev = rte_eth_dev_allocate(name);
2129         if (!eth_dev)
2130                 return -ENOMEM;
2131
2132         priv = mrvl_priv_create(name);
2133         if (!priv) {
2134                 ret = -ENOMEM;
2135                 goto out_free_dev;
2136         }
2137
2138         eth_dev->data->mac_addrs =
2139                 rte_zmalloc("mac_addrs",
2140                             ETHER_ADDR_LEN * MRVL_MAC_ADDRS_MAX, 0);
2141         if (!eth_dev->data->mac_addrs) {
2142                 RTE_LOG(ERR, PMD, "Failed to allocate space for eth addrs\n");
2143                 ret = -ENOMEM;
2144                 goto out_free_priv;
2145         }
2146
2147         memset(&req, 0, sizeof(req));
2148         strcpy(req.ifr_name, name);
2149         ret = ioctl(fd, SIOCGIFHWADDR, &req);
2150         if (ret)
2151                 goto out_free_mac;
2152
2153         memcpy(eth_dev->data->mac_addrs[0].addr_bytes,
2154                req.ifr_addr.sa_data, ETHER_ADDR_LEN);
2155
2156         eth_dev->rx_pkt_burst = mrvl_rx_pkt_burst;
2157         eth_dev->tx_pkt_burst = mrvl_tx_pkt_burst;
2158         eth_dev->data->kdrv = RTE_KDRV_NONE;
2159         eth_dev->data->dev_private = priv;
2160         eth_dev->device = &vdev->device;
2161         eth_dev->dev_ops = &mrvl_ops;
2162
2163         return 0;
2164 out_free_mac:
2165         rte_free(eth_dev->data->mac_addrs);
2166 out_free_dev:
2167         rte_eth_dev_release_port(eth_dev);
2168 out_free_priv:
2169         rte_free(priv);
2170
2171         return ret;
2172 }
2173
2174 /**
2175  * Cleanup previously created device representing Ethernet port.
2176  *
2177  * @param name
2178  *   Pointer to the port name.
2179  */
2180 static void
2181 mrvl_eth_dev_destroy(const char *name)
2182 {
2183         struct rte_eth_dev *eth_dev;
2184         struct mrvl_priv *priv;
2185
2186         eth_dev = rte_eth_dev_allocated(name);
2187         if (!eth_dev)
2188                 return;
2189
2190         priv = eth_dev->data->dev_private;
2191         pp2_bpool_deinit(priv->bpool);
2192         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
2193         rte_free(priv);
2194         rte_free(eth_dev->data->mac_addrs);
2195         rte_eth_dev_release_port(eth_dev);
2196 }
2197
2198 /**
2199  * Callback used by rte_kvargs_process() during argument parsing.
2200  *
2201  * @param key
2202  *   Pointer to the parsed key (unused).
2203  * @param value
2204  *   Pointer to the parsed value.
2205  * @param extra_args
2206  *   Pointer to the extra arguments which contains address of the
2207  *   table of pointers to parsed interface names.
2208  *
2209  * @return
2210  *   Always 0.
2211  */
2212 static int
2213 mrvl_get_ifnames(const char *key __rte_unused, const char *value,
2214                  void *extra_args)
2215 {
2216         struct mrvl_ifnames *ifnames = extra_args;
2217
2218         ifnames->names[ifnames->idx++] = value;
2219
2220         return 0;
2221 }
2222
2223 /**
2224  * Deinitialize per-lcore MUSDK hardware interfaces (hifs).
2225  */
2226 static void
2227 mrvl_deinit_hifs(void)
2228 {
2229         int i;
2230
2231         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++) {
2232                 if (hifs[i])
2233                         pp2_hif_deinit(hifs[i]);
2234         }
2235         used_hifs = MRVL_MUSDK_HIFS_RESERVED;
2236         memset(hifs, 0, sizeof(hifs));
2237 }
2238
2239 /**
2240  * DPDK callback to register the virtual device.
2241  *
2242  * @param vdev
2243  *   Pointer to the virtual device.
2244  *
2245  * @return
2246  *   0 on success, negative error value otherwise.
2247  */
2248 static int
2249 rte_pmd_mrvl_probe(struct rte_vdev_device *vdev)
2250 {
2251         struct rte_kvargs *kvlist;
2252         struct mrvl_ifnames ifnames;
2253         int ret = -EINVAL;
2254         uint32_t i, ifnum, cfgnum;
2255         const char *params;
2256
2257         params = rte_vdev_device_args(vdev);
2258         if (!params)
2259                 return -EINVAL;
2260
2261         kvlist = rte_kvargs_parse(params, valid_args);
2262         if (!kvlist)
2263                 return -EINVAL;
2264
2265         ifnum = rte_kvargs_count(kvlist, MRVL_IFACE_NAME_ARG);
2266         if (ifnum > RTE_DIM(ifnames.names))
2267                 goto out_free_kvlist;
2268
2269         ifnames.idx = 0;
2270         rte_kvargs_process(kvlist, MRVL_IFACE_NAME_ARG,
2271                            mrvl_get_ifnames, &ifnames);
2272
2273
2274         /*
2275          * The below system initialization should be done only once,
2276          * on the first provided configuration file
2277          */
2278         if (!mrvl_qos_cfg) {
2279                 cfgnum = rte_kvargs_count(kvlist, MRVL_CFG_ARG);
2280                 RTE_LOG(INFO, PMD, "Parsing config file!\n");
2281                 if (cfgnum > 1) {
2282                         RTE_LOG(ERR, PMD, "Cannot handle more than one config file!\n");
2283                         goto out_free_kvlist;
2284                 } else if (cfgnum == 1) {
2285                         rte_kvargs_process(kvlist, MRVL_CFG_ARG,
2286                                            mrvl_get_qoscfg, &mrvl_qos_cfg);
2287                 }
2288         }
2289
2290         if (mrvl_dev_num)
2291                 goto init_devices;
2292
2293         RTE_LOG(INFO, PMD, "Perform MUSDK initializations\n");
2294         /*
2295          * ret == -EEXIST is correct, it means DMA
2296          * has been already initialized (by another PMD).
2297          */
2298         ret = mv_sys_dma_mem_init(MRVL_MUSDK_DMA_MEMSIZE);
2299         if (ret < 0) {
2300                 if (ret != -EEXIST)
2301                         goto out_free_kvlist;
2302                 else
2303                         RTE_LOG(INFO, PMD,
2304                                 "DMA memory has been already initialized by a different driver.\n");
2305         }
2306
2307         ret = mrvl_init_pp2();
2308         if (ret) {
2309                 RTE_LOG(ERR, PMD, "Failed to init PP!\n");
2310                 goto out_deinit_dma;
2311         }
2312
2313         memset(mrvl_port_bpool_size, 0, sizeof(mrvl_port_bpool_size));
2314
2315         mrvl_lcore_first = RTE_MAX_LCORE;
2316         mrvl_lcore_last = 0;
2317
2318 init_devices:
2319         for (i = 0; i < ifnum; i++) {
2320                 RTE_LOG(INFO, PMD, "Creating %s\n", ifnames.names[i]);
2321                 ret = mrvl_eth_dev_create(vdev, ifnames.names[i]);
2322                 if (ret)
2323                         goto out_cleanup;
2324         }
2325         mrvl_dev_num += ifnum;
2326
2327         rte_kvargs_free(kvlist);
2328
2329         return 0;
2330 out_cleanup:
2331         for (; i > 0; i--)
2332                 mrvl_eth_dev_destroy(ifnames.names[i]);
2333
2334         if (mrvl_dev_num == 0)
2335                 mrvl_deinit_pp2();
2336 out_deinit_dma:
2337         if (mrvl_dev_num == 0)
2338                 mv_sys_dma_mem_destroy();
2339 out_free_kvlist:
2340         rte_kvargs_free(kvlist);
2341
2342         return ret;
2343 }
2344
2345 /**
2346  * DPDK callback to remove virtual device.
2347  *
2348  * @param vdev
2349  *   Pointer to the removed virtual device.
2350  *
2351  * @return
2352  *   0 on success, negative error value otherwise.
2353  */
2354 static int
2355 rte_pmd_mrvl_remove(struct rte_vdev_device *vdev)
2356 {
2357         int i;
2358         const char *name;
2359
2360         name = rte_vdev_device_name(vdev);
2361         if (!name)
2362                 return -EINVAL;
2363
2364         RTE_LOG(INFO, PMD, "Removing %s\n", name);
2365
2366         for (i = 0; i < rte_eth_dev_count(); i++) {
2367                 char ifname[RTE_ETH_NAME_MAX_LEN];
2368
2369                 rte_eth_dev_get_name_by_port(i, ifname);
2370                 mrvl_eth_dev_destroy(ifname);
2371                 mrvl_dev_num--;
2372         }
2373
2374         if (mrvl_dev_num == 0) {
2375                 RTE_LOG(INFO, PMD, "Perform MUSDK deinit\n");
2376                 mrvl_deinit_hifs();
2377                 mrvl_deinit_pp2();
2378                 mv_sys_dma_mem_destroy();
2379         }
2380
2381         return 0;
2382 }
2383
2384 static struct rte_vdev_driver pmd_mrvl_drv = {
2385         .probe = rte_pmd_mrvl_probe,
2386         .remove = rte_pmd_mrvl_remove,
2387 };
2388
2389 RTE_PMD_REGISTER_VDEV(net_mrvl, pmd_mrvl_drv);
2390 RTE_PMD_REGISTER_ALIAS(net_mrvl, eth_mrvl);