drivers/net/mrvl/mrvl_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2017 Marvell International Ltd.
   5  *   Copyright(c) 2017 Semihalf.
   6  *   All rights reserved.
   7  *
   8  *   Redistribution and use in source and binary forms, with or without
   9  *   modification, are permitted provided that the following conditions
  10  *   are met:
  11  *
  12  *     * Redistributions of source code must retain the above copyright
  13  *       notice, this list of conditions and the following disclaimer.
  14  *     * Redistributions in binary form must reproduce the above copyright
  15  *       notice, this list of conditions and the following disclaimer in
  16  *       the documentation and/or other materials provided with the
  17  *       distribution.
  18  *     * Neither the name of Semihalf nor the names of its
  19  *       contributors may be used to endorse or promote products derived
  20  *       from this software without specific prior written permission.
  21  *
  22  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  23  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  24  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  25  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  26  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  27  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  28  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  29  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  30  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  31  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  32  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  33  */
  34
  35 #include <rte_ethdev.h>
  36 #include <rte_kvargs.h>
  37 #include <rte_log.h>
  38 #include <rte_malloc.h>
  39 #include <rte_bus_vdev.h>
  40
  41 /* Unluckily, container_of is defined by both DPDK and MUSDK,
  42  * we'll declare only one version.
  43  *
  44  * Note that it is not used in this PMD anyway.
  45  */
  46 #ifdef container_of
  47 #undef container_of
  48 #endif
  49
  50 #include <drivers/mv_pp2.h>
  51 #include <drivers/mv_pp2_bpool.h>
  52 #include <drivers/mv_pp2_hif.h>
  53
  54 #include <fcntl.h>
  55 #include <linux/ethtool.h>
  56 #include <linux/sockios.h>
  57 #include <net/if.h>
  58 #include <net/if_arp.h>
  59 #include <sys/ioctl.h>
  60 #include <sys/socket.h>
  61 #include <sys/stat.h>
  62 #include <sys/types.h>
  63
  64 #include "mrvl_ethdev.h"
  65 #include "mrvl_qos.h"
  66
  67 /* bitmask with reserved hifs */
  68 #define MRVL_MUSDK_HIFS_RESERVED 0x0F
  69 /* bitmask with reserved bpools */
  70 #define MRVL_MUSDK_BPOOLS_RESERVED 0x07
  71 /* bitmask with reserved kernel RSS tables */
  72 #define MRVL_MUSDK_RSS_RESERVED 0x01
  73 /* maximum number of available hifs */
  74 #define MRVL_MUSDK_HIFS_MAX 9
  75
  76 /* prefetch shift */
  77 #define MRVL_MUSDK_PREFETCH_SHIFT 2
  78
  79 /* TCAM has 25 entries reserved for uc/mc filter entries */
  80 #define MRVL_MAC_ADDRS_MAX 25
  81 #define MRVL_MATCH_LEN 16
  82 #define MRVL_PKT_EFFEC_OFFS (MRVL_PKT_OFFS + MV_MH_SIZE)
  83 /* Maximum allowable packet size */
  84 #define MRVL_PKT_SIZE_MAX (10240 - MV_MH_SIZE)
  85
  86 #define MRVL_IFACE_NAME_ARG "iface"
  87 #define MRVL_CFG_ARG "cfg"
  88
  89 #define MRVL_BURST_SIZE 64
  90
  91 #define MRVL_ARP_LENGTH 28
  92
  93 #define MRVL_COOKIE_ADDR_INVALID ~0ULL
  94
  95 #define MRVL_COOKIE_HIGH_ADDR_SHIFT     (sizeof(pp2_cookie_t) * 8)
  96 #define MRVL_COOKIE_HIGH_ADDR_MASK      (~0ULL << MRVL_COOKIE_HIGH_ADDR_SHIFT)
  97
  98 /* Memory size (in bytes) for MUSDK dma buffers */
  99 #define MRVL_MUSDK_DMA_MEMSIZE 41943040
 100
 101 static const char * const valid_args[] = {
 102         MRVL_IFACE_NAME_ARG,
 103         MRVL_CFG_ARG,
 104         NULL
 105 };
 106
 107 static int used_hifs = MRVL_MUSDK_HIFS_RESERVED;
 108 static struct pp2_hif *hifs[RTE_MAX_LCORE];
 109 static int used_bpools[PP2_NUM_PKT_PROC] = {
 110         MRVL_MUSDK_BPOOLS_RESERVED,
 111         MRVL_MUSDK_BPOOLS_RESERVED
 112 };
 113
 114 struct pp2_bpool *mrvl_port_to_bpool_lookup[RTE_MAX_ETHPORTS];
 115 int mrvl_port_bpool_size[PP2_NUM_PKT_PROC][PP2_BPOOL_NUM_POOLS][RTE_MAX_LCORE];
 116 uint64_t cookie_addr_high = MRVL_COOKIE_ADDR_INVALID;
 117
 118 /*
 119  * To use buffer harvesting based on loopback port shadow queue structure
 120  * was introduced for buffers information bookkeeping.
 121  *
 122  * Before sending the packet, related buffer information (pp2_buff_inf) is
 123  * stored in shadow queue. After packet is transmitted no longer used
 124  * packet buffer is released back to it's original hardware pool,
 125  * on condition it originated from interface.
 126  * In case it  was generated by application itself i.e: mbuf->port field is
 127  * 0xff then its released to software mempool.
 128  */
 129 struct mrvl_shadow_txq {
 130         int head;           /* write index - used when sending buffers */
 131         int tail;           /* read index - used when releasing buffers */
 132         u16 size;           /* queue occupied size */
 133         u16 num_to_release; /* number of buffers sent, that can be released */
 134         struct buff_release_entry ent[MRVL_PP2_TX_SHADOWQ_SIZE]; /* q entries */
 135 };
 136
 137 struct mrvl_rxq {
 138         struct mrvl_priv *priv;
 139         struct rte_mempool *mp;
 140         int queue_id;
 141         int port_id;
 142         int cksum_enabled;
 143         uint64_t bytes_recv;
 144         uint64_t drop_mac;
 145 };
 146
 147 struct mrvl_txq {
 148         struct mrvl_priv *priv;
 149         int queue_id;
 150         int port_id;
 151         uint64_t bytes_sent;
 152 };
 153
 154 /*
 155  * Every tx queue should have dedicated shadow tx queue.
 156  *
 157  * Ports assigned by DPDK might not start at zero or be continuous so
 158  * as a workaround define shadow queues for each possible port so that
 159  * we eventually fit somewhere.
 160  */
 161 struct mrvl_shadow_txq shadow_txqs[RTE_MAX_ETHPORTS][RTE_MAX_LCORE];
 162
 163 /** Number of ports configured. */
 164 int mrvl_ports_nb;
 165 static int mrvl_lcore_first;
 166 static int mrvl_lcore_last;
 167
 168 static inline int
 169 mrvl_get_bpool_size(int pp2_id, int pool_id)
 170 {
 171         int i;
 172         int size = 0;
 173
 174         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++)
 175                 size += mrvl_port_bpool_size[pp2_id][pool_id][i];
 176
 177         return size;
 178 }
 179
 180 static inline int
 181 mrvl_reserve_bit(int *bitmap, int max)
 182 {
 183         int n = sizeof(*bitmap) * 8 - __builtin_clz(*bitmap);
 184
 185         if (n >= max)
 186                 return -1;
 187
 188         *bitmap |= 1 << n;
 189
 190         return n;
 191 }
 192
 193 /**
 194  * Configure rss based on dpdk rss configuration.
 195  *
 196  * @param priv
 197  *   Pointer to private structure.
 198  * @param rss_conf
 199  *   Pointer to RSS configuration.
 200  *
 201  * @return
 202  *   0 on success, negative error value otherwise.
 203  */
 204 static int
 205 mrvl_configure_rss(struct mrvl_priv *priv, struct rte_eth_rss_conf *rss_conf)
 206 {
 207         if (rss_conf->rss_key)
 208                 RTE_LOG(WARNING, PMD, "Changing hash key is not supported\n");
 209
 210         if (rss_conf->rss_hf == 0) {
 211                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 212         } else if (rss_conf->rss_hf & ETH_RSS_IPV4) {
 213                 priv->ppio_params.inqs_params.hash_type =
 214                         PP2_PPIO_HASH_T_2_TUPLE;
 215         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) {
 216                 priv->ppio_params.inqs_params.hash_type =
 217                         PP2_PPIO_HASH_T_5_TUPLE;
 218                 priv->rss_hf_tcp = 1;
 219         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
 220                 priv->ppio_params.inqs_params.hash_type =
 221                         PP2_PPIO_HASH_T_5_TUPLE;
 222                 priv->rss_hf_tcp = 0;
 223         } else {
 224                 return -EINVAL;
 225         }
 226
 227         return 0;
 228 }
 229
 230 /**
 231  * Ethernet device configuration.
 232  *
 233  * Prepare the driver for a given number of TX and RX queues and
 234  * configure RSS.
 235  *
 236  * @param dev
 237  *   Pointer to Ethernet device structure.
 238  *
 239  * @return
 240  *   0 on success, negative error value otherwise.
 241  */
 242 static int
 243 mrvl_dev_configure(struct rte_eth_dev *dev)
 244 {
 245         struct mrvl_priv *priv = dev->data->dev_private;
 246         int ret;
 247
 248         if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_NONE &&
 249             dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) {
 250                 RTE_LOG(INFO, PMD, "Unsupported rx multi queue mode %d\n",
 251                         dev->data->dev_conf.rxmode.mq_mode);
 252                 return -EINVAL;
 253         }
 254
 255         if (!dev->data->dev_conf.rxmode.hw_strip_crc) {
 256                 RTE_LOG(INFO, PMD,
 257                         "L2 CRC stripping is always enabled in hw\n");
 258                 dev->data->dev_conf.rxmode.hw_strip_crc = 1;
 259         }
 260
 261         if (dev->data->dev_conf.rxmode.hw_vlan_strip) {
 262                 RTE_LOG(INFO, PMD, "VLAN stripping not supported\n");
 263                 return -EINVAL;
 264         }
 265
 266         if (dev->data->dev_conf.rxmode.split_hdr_size) {
 267                 RTE_LOG(INFO, PMD, "Split headers not supported\n");
 268                 return -EINVAL;
 269         }
 270
 271         if (dev->data->dev_conf.rxmode.enable_scatter) {
 272                 RTE_LOG(INFO, PMD, "RX Scatter/Gather not supported\n");
 273                 return -EINVAL;
 274         }
 275
 276         if (dev->data->dev_conf.rxmode.enable_lro) {
 277                 RTE_LOG(INFO, PMD, "LRO not supported\n");
 278                 return -EINVAL;
 279         }
 280
 281         if (dev->data->dev_conf.rxmode.jumbo_frame)
 282                 dev->data->mtu = dev->data->dev_conf.rxmode.max_rx_pkt_len -
 283                                  ETHER_HDR_LEN - ETHER_CRC_LEN;
 284
 285         ret = mrvl_configure_rxqs(priv, dev->data->port_id,
 286                                   dev->data->nb_rx_queues);
 287         if (ret < 0)
 288                 return ret;
 289
 290         priv->ppio_params.outqs_params.num_outqs = dev->data->nb_tx_queues;
 291         priv->ppio_params.maintain_stats = 1;
 292         priv->nb_rx_queues = dev->data->nb_rx_queues;
 293
 294         if (dev->data->nb_rx_queues == 1 &&
 295             dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) {
 296                 RTE_LOG(WARNING, PMD, "Disabling hash for 1 rx queue\n");
 297                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 298
 299                 return 0;
 300         }
 301
 302         return mrvl_configure_rss(priv,
 303                                   &dev->data->dev_conf.rx_adv_conf.rss_conf);
 304 }
 305
 306 /**
 307  * DPDK callback to change the MTU.
 308  *
 309  * Setting the MTU affects hardware MRU (packets larger than the MRU
 310  * will be dropped).
 311  *
 312  * @param dev
 313  *   Pointer to Ethernet device structure.
 314  * @param mtu
 315  *   New MTU.
 316  *
 317  * @return
 318  *   0 on success, negative error value otherwise.
 319  */
 320 static int
 321 mrvl_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 322 {
 323         struct mrvl_priv *priv = dev->data->dev_private;
 324         /* extra MV_MH_SIZE bytes are required for Marvell tag */
 325         uint16_t mru = mtu + MV_MH_SIZE + ETHER_HDR_LEN + ETHER_CRC_LEN;
 326         int ret;
 327
 328         if (mtu < ETHER_MIN_MTU || mru > MRVL_PKT_SIZE_MAX)
 329                 return -EINVAL;
 330
 331         ret = pp2_ppio_set_mru(priv->ppio, mru);
 332         if (ret)
 333                 return ret;
 334
 335         return pp2_ppio_set_mtu(priv->ppio, mtu);
 336 }
 337
 338 /**
 339  * DPDK callback to bring the link up.
 340  *
 341  * @param dev
 342  *   Pointer to Ethernet device structure.
 343  *
 344  * @return
 345  *   0 on success, negative error value otherwise.
 346  */
 347 static int
 348 mrvl_dev_set_link_up(struct rte_eth_dev *dev)
 349 {
 350         struct mrvl_priv *priv = dev->data->dev_private;
 351         int ret;
 352
 353         ret = pp2_ppio_enable(priv->ppio);
 354         if (ret)
 355                 return ret;
 356
 357         /*
 358          * mtu/mru can be updated if pp2_ppio_enable() was called at least once
 359          * as pp2_ppio_enable() changes port->t_mode from default 0 to
 360          * PP2_TRAFFIC_INGRESS_EGRESS.
 361          *
 362          * Set mtu to default DPDK value here.
 363          */
 364         ret = mrvl_mtu_set(dev, dev->data->mtu);
 365         if (ret)
 366                 pp2_ppio_disable(priv->ppio);
 367
 368         dev->data->dev_link.link_status = ETH_LINK_UP;
 369
 370         return ret;
 371 }
 372
 373 /**
 374  * DPDK callback to bring the link down.
 375  *
 376  * @param dev
 377  *   Pointer to Ethernet device structure.
 378  *
 379  * @return
 380  *   0 on success, negative error value otherwise.
 381  */
 382 static int
 383 mrvl_dev_set_link_down(struct rte_eth_dev *dev)
 384 {
 385         struct mrvl_priv *priv = dev->data->dev_private;
 386         int ret;
 387
 388         ret = pp2_ppio_disable(priv->ppio);
 389         if (ret)
 390                 return ret;
 391
 392         dev->data->dev_link.link_status = ETH_LINK_DOWN;
 393
 394         return ret;
 395 }
 396
 397 /**
 398  * DPDK callback to start the device.
 399  *
 400  * @param dev
 401  *   Pointer to Ethernet device structure.
 402  *
 403  * @return
 404  *   0 on success, negative errno value on failure.
 405  */
 406 static int
 407 mrvl_dev_start(struct rte_eth_dev *dev)
 408 {
 409         struct mrvl_priv *priv = dev->data->dev_private;
 410         char match[MRVL_MATCH_LEN];
 411         int ret;
 412
 413         snprintf(match, sizeof(match), "ppio-%d:%d",
 414                  priv->pp_id, priv->ppio_id);
 415         priv->ppio_params.match = match;
 416
 417         /*
 418          * Calculate the maximum bpool size for refill feature to 1.5 of the
 419          * configured size. In case the bpool size will exceed this value,
 420          * superfluous buffers will be removed
 421          */
 422         priv->bpool_max_size = priv->bpool_init_size +
 423                               (priv->bpool_init_size >> 1);
 424         /*
 425          * Calculate the minimum bpool size for refill feature as follows:
 426          * 2 default burst sizes multiply by number of rx queues.
 427          * If the bpool size will be below this value, new buffers will
 428          * be added to the pool.
 429          */
 430         priv->bpool_min_size = priv->nb_rx_queues * MRVL_BURST_SIZE * 2;
 431
 432         ret = pp2_ppio_init(&priv->ppio_params, &priv->ppio);
 433         if (ret)
 434                 return ret;
 435
 436         /*
 437          * In case there are some some stale uc/mc mac addresses flush them
 438          * here. It cannot be done during mrvl_dev_close() as port information
 439          * is already gone at that point (due to pp2_ppio_deinit() in
 440          * mrvl_dev_stop()).
 441          */
 442         if (!priv->uc_mc_flushed) {
 443                 ret = pp2_ppio_flush_mac_addrs(priv->ppio, 1, 1);
 444                 if (ret) {
 445                         RTE_LOG(ERR, PMD,
 446                                 "Failed to flush uc/mc filter list\n");
 447                         goto out;
 448                 }
 449                 priv->uc_mc_flushed = 1;
 450         }
 451
 452         if (!priv->vlan_flushed) {
 453                 ret = pp2_ppio_flush_vlan(priv->ppio);
 454                 if (ret) {
 455                         RTE_LOG(ERR, PMD, "Failed to flush vlan list\n");
 456                         /*
 457                          * TODO
 458                          * once pp2_ppio_flush_vlan() is supported jump to out
 459                          * goto out;
 460                          */
 461                 }
 462                 priv->vlan_flushed = 1;
 463         }
 464
 465         /* For default QoS config, don't start classifier. */
 466         if (mrvl_qos_cfg) {
 467                 ret = mrvl_start_qos_mapping(priv);
 468                 if (ret) {
 469                         pp2_ppio_deinit(priv->ppio);
 470                         return ret;
 471                 }
 472         }
 473
 474         ret = mrvl_dev_set_link_up(dev);
 475         if (ret)
 476                 goto out;
 477
 478         return 0;
 479 out:
 480         pp2_ppio_deinit(priv->ppio);
 481         return ret;
 482 }
 483
 484 /**
 485  * Flush receive queues.
 486  *
 487  * @param dev
 488  *   Pointer to Ethernet device structure.
 489  */
 490 static void
 491 mrvl_flush_rx_queues(struct rte_eth_dev *dev)
 492 {
 493         int i;
 494
 495         RTE_LOG(INFO, PMD, "Flushing rx queues\n");
 496         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 497                 int ret, num;
 498
 499                 do {
 500                         struct mrvl_rxq *q = dev->data->rx_queues[i];
 501                         struct pp2_ppio_desc descs[MRVL_PP2_RXD_MAX];
 502
 503                         num = MRVL_PP2_RXD_MAX;
 504                         ret = pp2_ppio_recv(q->priv->ppio,
 505                                             q->priv->rxq_map[q->queue_id].tc,
 506                                             q->priv->rxq_map[q->queue_id].inq,
 507                                             descs, (uint16_t *)&num);
 508                 } while (ret == 0 && num);
 509         }
 510 }
 511
 512 /**
 513  * Flush transmit shadow queues.
 514  *
 515  * @param dev
 516  *   Pointer to Ethernet device structure.
 517  */
 518 static void
 519 mrvl_flush_tx_shadow_queues(struct rte_eth_dev *dev)
 520 {
 521         int i;
 522
 523         RTE_LOG(INFO, PMD, "Flushing tx shadow queues\n");
 524         for (i = 0; i < RTE_MAX_LCORE; i++) {
 525                 struct mrvl_shadow_txq *sq =
 526                         &shadow_txqs[dev->data->port_id][i];
 527
 528                 while (sq->tail != sq->head) {
 529                         uint64_t addr = cookie_addr_high |
 530                                         sq->ent[sq->tail].buff.cookie;
 531                         rte_pktmbuf_free((struct rte_mbuf *)addr);
 532                         sq->tail = (sq->tail + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
 533                 }
 534
 535                 memset(sq, 0, sizeof(*sq));
 536         }
 537 }
 538
 539 /**
 540  * Flush hardware bpool (buffer-pool).
 541  *
 542  * @param dev
 543  *   Pointer to Ethernet device structure.
 544  */
 545 static void
 546 mrvl_flush_bpool(struct rte_eth_dev *dev)
 547 {
 548         struct mrvl_priv *priv = dev->data->dev_private;
 549         uint32_t num;
 550         int ret;
 551
 552         ret = pp2_bpool_get_num_buffs(priv->bpool, &num);
 553         if (ret) {
 554                 RTE_LOG(ERR, PMD, "Failed to get bpool buffers number\n");
 555                 return;
 556         }
 557
 558         while (num--) {
 559                 struct pp2_buff_inf inf;
 560                 uint64_t addr;
 561
 562                 ret = pp2_bpool_get_buff(hifs[rte_lcore_id()], priv->bpool,
 563                                          &inf);
 564                 if (ret)
 565                         break;
 566
 567                 addr = cookie_addr_high | inf.cookie;
 568                 rte_pktmbuf_free((struct rte_mbuf *)addr);
 569         }
 570 }
 571
 572 /**
 573  * DPDK callback to stop the device.
 574  *
 575  * @param dev
 576  *   Pointer to Ethernet device structure.
 577  */
 578 static void
 579 mrvl_dev_stop(struct rte_eth_dev *dev)
 580 {
 581         struct mrvl_priv *priv = dev->data->dev_private;
 582
 583         mrvl_dev_set_link_down(dev);
 584         mrvl_flush_rx_queues(dev);
 585         mrvl_flush_tx_shadow_queues(dev);
 586         if (priv->qos_tbl)
 587                 pp2_cls_qos_tbl_deinit(priv->qos_tbl);
 588         pp2_ppio_deinit(priv->ppio);
 589         priv->ppio = NULL;
 590 }
 591
 592 /**
 593  * DPDK callback to close the device.
 594  *
 595  * @param dev
 596  *   Pointer to Ethernet device structure.
 597  */
 598 static void
 599 mrvl_dev_close(struct rte_eth_dev *dev)
 600 {
 601         struct mrvl_priv *priv = dev->data->dev_private;
 602         size_t i;
 603
 604         for (i = 0; i < priv->ppio_params.inqs_params.num_tcs; ++i) {
 605                 struct pp2_ppio_tc_params *tc_params =
 606                         &priv->ppio_params.inqs_params.tcs_params[i];
 607
 608                 if (tc_params->inqs_params) {
 609                         rte_free(tc_params->inqs_params);
 610                         tc_params->inqs_params = NULL;
 611                 }
 612         }
 613
 614         mrvl_flush_bpool(dev);
 615 }
 616
 617 /**
 618  * DPDK callback to retrieve physical link information.
 619  *
 620  * @param dev
 621  *   Pointer to Ethernet device structure.
 622  * @param wait_to_complete
 623  *   Wait for request completion (ignored).
 624  *
 625  * @return
 626  *   0 on success, negative error value otherwise.
 627  */
 628 static int
 629 mrvl_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused)
 630 {
 631         /*
 632          * TODO
 633          * once MUSDK provides necessary API use it here
 634          */
 635         struct ethtool_cmd edata;
 636         struct ifreq req;
 637         int ret, fd;
 638
 639         edata.cmd = ETHTOOL_GSET;
 640
 641         strcpy(req.ifr_name, dev->data->name);
 642         req.ifr_data = (void *)&edata;
 643
 644         fd = socket(AF_INET, SOCK_DGRAM, 0);
 645         if (fd == -1)
 646                 return -EFAULT;
 647
 648         ret = ioctl(fd, SIOCETHTOOL, &req);
 649         if (ret == -1) {
 650                 close(fd);
 651                 return -EFAULT;
 652         }
 653
 654         close(fd);
 655
 656         switch (ethtool_cmd_speed(&edata)) {
 657         case SPEED_10:
 658                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10M;
 659                 break;
 660         case SPEED_100:
 661                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_100M;
 662                 break;
 663         case SPEED_1000:
 664                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_1G;
 665                 break;
 666         case SPEED_10000:
 667                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10G;
 668                 break;
 669         default:
 670                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE;
 671         }
 672
 673         dev->data->dev_link.link_duplex = edata.duplex ? ETH_LINK_FULL_DUPLEX :
 674                                                          ETH_LINK_HALF_DUPLEX;
 675         dev->data->dev_link.link_autoneg = edata.autoneg ? ETH_LINK_AUTONEG :
 676                                                            ETH_LINK_FIXED;
 677
 678         return 0;
 679 }
 680
 681 /**
 682  * DPDK callback to enable promiscuous mode.
 683  *
 684  * @param dev
 685  *   Pointer to Ethernet device structure.
 686  */
 687 static void
 688 mrvl_promiscuous_enable(struct rte_eth_dev *dev)
 689 {
 690         struct mrvl_priv *priv = dev->data->dev_private;
 691         int ret;
 692
 693         ret = pp2_ppio_set_uc_promisc(priv->ppio, 1);
 694         if (ret)
 695                 RTE_LOG(ERR, PMD, "Failed to enable promiscuous mode\n");
 696 }
 697
 698 /**
 699  * DPDK callback to enable allmulti mode.
 700  *
 701  * @param dev
 702  *   Pointer to Ethernet device structure.
 703  */
 704 static void
 705 mrvl_allmulticast_enable(struct rte_eth_dev *dev)
 706 {
 707         struct mrvl_priv *priv = dev->data->dev_private;
 708         int ret;
 709
 710         ret = pp2_ppio_set_mc_promisc(priv->ppio, 1);
 711         if (ret)
 712                 RTE_LOG(ERR, PMD, "Failed enable all-multicast mode\n");
 713 }
 714
 715 /**
 716  * DPDK callback to disable promiscuous mode.
 717  *
 718  * @param dev
 719  *   Pointer to Ethernet device structure.
 720  */
 721 static void
 722 mrvl_promiscuous_disable(struct rte_eth_dev *dev)
 723 {
 724         struct mrvl_priv *priv = dev->data->dev_private;
 725         int ret;
 726
 727         ret = pp2_ppio_set_uc_promisc(priv->ppio, 0);
 728         if (ret)
 729                 RTE_LOG(ERR, PMD, "Failed to disable promiscuous mode\n");
 730 }
 731
 732 /**
 733  * DPDK callback to disable allmulticast mode.
 734  *
 735  * @param dev
 736  *   Pointer to Ethernet device structure.
 737  */
 738 static void
 739 mrvl_allmulticast_disable(struct rte_eth_dev *dev)
 740 {
 741         struct mrvl_priv *priv = dev->data->dev_private;
 742         int ret;
 743
 744         ret = pp2_ppio_set_mc_promisc(priv->ppio, 0);
 745         if (ret)
 746                 RTE_LOG(ERR, PMD, "Failed to disable all-multicast mode\n");
 747 }
 748
 749 /**
 750  * DPDK callback to remove a MAC address.
 751  *
 752  * @param dev
 753  *   Pointer to Ethernet device structure.
 754  * @param index
 755  *   MAC address index.
 756  */
 757 static void
 758 mrvl_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
 759 {
 760         struct mrvl_priv *priv = dev->data->dev_private;
 761         char buf[ETHER_ADDR_FMT_SIZE];
 762         int ret;
 763
 764         ret = pp2_ppio_remove_mac_addr(priv->ppio,
 765                                        dev->data->mac_addrs[index].addr_bytes);
 766         if (ret) {
 767                 ether_format_addr(buf, sizeof(buf),
 768                                   &dev->data->mac_addrs[index]);
 769                 RTE_LOG(ERR, PMD, "Failed to remove mac %s\n", buf);
 770         }
 771 }
 772
 773 /**
 774  * DPDK callback to add a MAC address.
 775  *
 776  * @param dev
 777  *   Pointer to Ethernet device structure.
 778  * @param mac_addr
 779  *   MAC address to register.
 780  * @param index
 781  *   MAC address index.
 782  * @param vmdq
 783  *   VMDq pool index to associate address with (unused).
 784  *
 785  * @return
 786  *   0 on success, negative error value otherwise.
 787  */
 788 static int
 789 mrvl_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
 790                   uint32_t index, uint32_t vmdq __rte_unused)
 791 {
 792         struct mrvl_priv *priv = dev->data->dev_private;
 793         char buf[ETHER_ADDR_FMT_SIZE];
 794         int ret;
 795
 796         if (index == 0)
 797                 /* For setting index 0, mrvl_mac_addr_set() should be used.*/
 798                 return -1;
 799
 800         /*
 801          * Maximum number of uc addresses can be tuned via kernel module mvpp2x
 802          * parameter uc_filter_max. Maximum number of mc addresses is then
 803          * MRVL_MAC_ADDRS_MAX - uc_filter_max. Currently it defaults to 4 and
 804          * 21 respectively.
 805          *
 806          * If more than uc_filter_max uc addresses were added to filter list
 807          * then NIC will switch to promiscuous mode automatically.
 808          *
 809          * If more than MRVL_MAC_ADDRS_MAX - uc_filter_max number mc addresses
 810          * were added to filter list then NIC will switch to all-multicast mode
 811          * automatically.
 812          */
 813         ret = pp2_ppio_add_mac_addr(priv->ppio, mac_addr->addr_bytes);
 814         if (ret) {
 815                 ether_format_addr(buf, sizeof(buf), mac_addr);
 816                 RTE_LOG(ERR, PMD, "Failed to add mac %s\n", buf);
 817                 return -1;
 818         }
 819
 820         return 0;
 821 }
 822
 823 /**
 824  * DPDK callback to set the primary MAC address.
 825  *
 826  * @param dev
 827  *   Pointer to Ethernet device structure.
 828  * @param mac_addr
 829  *   MAC address to register.
 830  */
 831 static void
 832 mrvl_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
 833 {
 834         struct mrvl_priv *priv = dev->data->dev_private;
 835
 836         pp2_ppio_set_mac_addr(priv->ppio, mac_addr->addr_bytes);
 837         /*
 838          * TODO
 839          * Port stops sending packets if pp2_ppio_set_mac_addr()
 840          * was called after pp2_ppio_enable(). As a quick fix issue
 841          * enable port once again.
 842          */
 843         pp2_ppio_enable(priv->ppio);
 844 }
 845
 846 /**
 847  * DPDK callback to get device statistics.
 848  *
 849  * @param dev
 850  *   Pointer to Ethernet device structure.
 851  * @param stats
 852  *   Stats structure output buffer.
 853  *
 854  * @return
 855  *   0 on success, negative error value otherwise.
 856  */
 857 static int
 858 mrvl_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 859 {
 860         struct mrvl_priv *priv = dev->data->dev_private;
 861         struct pp2_ppio_statistics ppio_stats;
 862         uint64_t drop_mac = 0;
 863         unsigned int i, idx, ret;
 864
 865         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 866                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
 867                 struct pp2_ppio_inq_statistics rx_stats;
 868
 869                 if (!rxq)
 870                         continue;
 871
 872                 idx = rxq->queue_id;
 873                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
 874                         RTE_LOG(ERR, PMD,
 875                                 "rx queue %d stats out of range (0 - %d)\n",
 876                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
 877                         continue;
 878                 }
 879
 880                 ret = pp2_ppio_inq_get_statistics(priv->ppio,
 881                                                   priv->rxq_map[idx].tc,
 882                                                   priv->rxq_map[idx].inq,
 883                                                   &rx_stats, 0);
 884                 if (unlikely(ret)) {
 885                         RTE_LOG(ERR, PMD,
 886                                 "Failed to update rx queue %d stats\n", idx);
 887                         break;
 888                 }
 889
 890                 stats->q_ibytes[idx] = rxq->bytes_recv;
 891                 stats->q_ipackets[idx] = rx_stats.enq_desc - rxq->drop_mac;
 892                 stats->q_errors[idx] = rx_stats.drop_early +
 893                                        rx_stats.drop_fullq +
 894                                        rx_stats.drop_bm +
 895                                        rxq->drop_mac;
 896                 stats->ibytes += rxq->bytes_recv;
 897                 drop_mac += rxq->drop_mac;
 898         }
 899
 900         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 901                 struct mrvl_txq *txq = dev->data->tx_queues[i];
 902                 struct pp2_ppio_outq_statistics tx_stats;
 903
 904                 if (!txq)
 905                         continue;
 906
 907                 idx = txq->queue_id;
 908                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
 909                         RTE_LOG(ERR, PMD,
 910                                 "tx queue %d stats out of range (0 - %d)\n",
 911                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
 912                 }
 913
 914                 ret = pp2_ppio_outq_get_statistics(priv->ppio, idx,
 915                                                    &tx_stats, 0);
 916                 if (unlikely(ret)) {
 917                         RTE_LOG(ERR, PMD,
 918                                 "Failed to update tx queue %d stats\n", idx);
 919                         break;
 920                 }
 921
 922                 stats->q_opackets[idx] = tx_stats.deq_desc;
 923                 stats->q_obytes[idx] = txq->bytes_sent;
 924                 stats->obytes += txq->bytes_sent;
 925         }
 926
 927         ret = pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
 928         if (unlikely(ret)) {
 929                 RTE_LOG(ERR, PMD, "Failed to update port statistics\n");
 930                 return ret;
 931         }
 932
 933         stats->ipackets += ppio_stats.rx_packets - drop_mac;
 934         stats->opackets += ppio_stats.tx_packets;
 935         stats->imissed += ppio_stats.rx_fullq_dropped +
 936                           ppio_stats.rx_bm_dropped +
 937                           ppio_stats.rx_early_dropped +
 938                           ppio_stats.rx_fifo_dropped +
 939                           ppio_stats.rx_cls_dropped;
 940         stats->ierrors = drop_mac;
 941
 942         return 0;
 943 }
 944
 945 /**
 946  * DPDK callback to clear device statistics.
 947  *
 948  * @param dev
 949  *   Pointer to Ethernet device structure.
 950  */
 951 static void
 952 mrvl_stats_reset(struct rte_eth_dev *dev)
 953 {
 954         struct mrvl_priv *priv = dev->data->dev_private;
 955         int i;
 956
 957         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 958                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
 959
 960                 pp2_ppio_inq_get_statistics(priv->ppio, priv->rxq_map[i].tc,
 961                                             priv->rxq_map[i].inq, NULL, 1);
 962                 rxq->bytes_recv = 0;
 963                 rxq->drop_mac = 0;
 964         }
 965
 966         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 967                 struct mrvl_txq *txq = dev->data->tx_queues[i];
 968
 969                 pp2_ppio_outq_get_statistics(priv->ppio, i, NULL, 1);
 970                 txq->bytes_sent = 0;
 971         }
 972
 973         pp2_ppio_get_statistics(priv->ppio, NULL, 1);
 974 }
 975
 976 /**
 977  * DPDK callback to get information about the device.
 978  *
 979  * @param dev
 980  *   Pointer to Ethernet device structure (unused).
 981  * @param info
 982  *   Info structure output buffer.
 983  */
 984 static void
 985 mrvl_dev_infos_get(struct rte_eth_dev *dev __rte_unused,
 986                    struct rte_eth_dev_info *info)
 987 {
 988         info->speed_capa = ETH_LINK_SPEED_10M |
 989                            ETH_LINK_SPEED_100M |
 990                            ETH_LINK_SPEED_1G |
 991                            ETH_LINK_SPEED_10G;
 992
 993         info->max_rx_queues = MRVL_PP2_RXQ_MAX;
 994         info->max_tx_queues = MRVL_PP2_TXQ_MAX;
 995         info->max_mac_addrs = MRVL_MAC_ADDRS_MAX;
 996
 997         info->rx_desc_lim.nb_max = MRVL_PP2_RXD_MAX;
 998         info->rx_desc_lim.nb_min = MRVL_PP2_RXD_MIN;
 999         info->rx_desc_lim.nb_align = MRVL_PP2_RXD_ALIGN;
1000
1001         info->tx_desc_lim.nb_max = MRVL_PP2_TXD_MAX;
1002         info->tx_desc_lim.nb_min = MRVL_PP2_TXD_MIN;
1003         info->tx_desc_lim.nb_align = MRVL_PP2_TXD_ALIGN;
1004
1005         info->rx_offload_capa = DEV_RX_OFFLOAD_JUMBO_FRAME |
1006                                 DEV_RX_OFFLOAD_VLAN_FILTER |
1007                                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1008                                 DEV_RX_OFFLOAD_UDP_CKSUM |
1009                                 DEV_RX_OFFLOAD_TCP_CKSUM;
1010
1011         info->tx_offload_capa = DEV_TX_OFFLOAD_IPV4_CKSUM |
1012                                 DEV_TX_OFFLOAD_UDP_CKSUM |
1013                                 DEV_TX_OFFLOAD_TCP_CKSUM;
1014
1015         info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1016                                        ETH_RSS_NONFRAG_IPV4_TCP |
1017                                        ETH_RSS_NONFRAG_IPV4_UDP;
1018
1019         /* By default packets are dropped if no descriptors are available */
1020         info->default_rxconf.rx_drop_en = 1;
1021
1022         info->max_rx_pktlen = MRVL_PKT_SIZE_MAX;
1023 }
1024
1025 /**
1026  * Return supported packet types.
1027  *
1028  * @param dev
1029  *   Pointer to Ethernet device structure (unused).
1030  *
1031  * @return
1032  *   Const pointer to the table with supported packet types.
1033  */
1034 static const uint32_t *
1035 mrvl_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1036 {
1037         static const uint32_t ptypes[] = {
1038                 RTE_PTYPE_L2_ETHER,
1039                 RTE_PTYPE_L3_IPV4,
1040                 RTE_PTYPE_L3_IPV4_EXT,
1041                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
1042                 RTE_PTYPE_L3_IPV6,
1043                 RTE_PTYPE_L3_IPV6_EXT,
1044                 RTE_PTYPE_L2_ETHER_ARP,
1045                 RTE_PTYPE_L4_TCP,
1046                 RTE_PTYPE_L4_UDP
1047         };
1048
1049         return ptypes;
1050 }
1051
1052 /**
1053  * DPDK callback to get information about specific receive queue.
1054  *
1055  * @param dev
1056  *   Pointer to Ethernet device structure.
1057  * @param rx_queue_id
1058  *   Receive queue index.
1059  * @param qinfo
1060  *   Receive queue information structure.
1061  */
1062 static void mrvl_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
1063                               struct rte_eth_rxq_info *qinfo)
1064 {
1065         struct mrvl_rxq *q = dev->data->rx_queues[rx_queue_id];
1066         struct mrvl_priv *priv = dev->data->dev_private;
1067         int inq = priv->rxq_map[rx_queue_id].inq;
1068         int tc = priv->rxq_map[rx_queue_id].tc;
1069         struct pp2_ppio_tc_params *tc_params =
1070                 &priv->ppio_params.inqs_params.tcs_params[tc];
1071
1072         qinfo->mp = q->mp;
1073         qinfo->nb_desc = tc_params->inqs_params[inq].size;
1074 }
1075
1076 /**
1077  * DPDK callback to get information about specific transmit queue.
1078  *
1079  * @param dev
1080  *   Pointer to Ethernet device structure.
1081  * @param tx_queue_id
1082  *   Transmit queue index.
1083  * @param qinfo
1084  *   Transmit queue information structure.
1085  */
1086 static void mrvl_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
1087                               struct rte_eth_txq_info *qinfo)
1088 {
1089         struct mrvl_priv *priv = dev->data->dev_private;
1090
1091         qinfo->nb_desc =
1092                 priv->ppio_params.outqs_params.outqs_params[tx_queue_id].size;
1093 }
1094
1095 /**
1096  * DPDK callback to Configure a VLAN filter.
1097  *
1098  * @param dev
1099  *   Pointer to Ethernet device structure.
1100  * @param vlan_id
1101  *   VLAN ID to filter.
1102  * @param on
1103  *   Toggle filter.
1104  *
1105  * @return
1106  *   0 on success, negative error value otherwise.
1107  */
1108 static int
1109 mrvl_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1110 {
1111         struct mrvl_priv *priv = dev->data->dev_private;
1112
1113         return on ? pp2_ppio_add_vlan(priv->ppio, vlan_id) :
1114                     pp2_ppio_remove_vlan(priv->ppio, vlan_id);
1115 }
1116
1117 /**
1118  * Release buffers to hardware bpool (buffer-pool)
1119  *
1120  * @param rxq
1121  *   Receive queue pointer.
1122  * @param num
1123  *   Number of buffers to release to bpool.
1124  *
1125  * @return
1126  *   0 on success, negative error value otherwise.
1127  */
1128 static int
1129 mrvl_fill_bpool(struct mrvl_rxq *rxq, int num)
1130 {
1131         struct buff_release_entry entries[MRVL_PP2_TXD_MAX];
1132         struct rte_mbuf *mbufs[MRVL_PP2_TXD_MAX];
1133         int i, ret;
1134         unsigned int core_id = rte_lcore_id();
1135         struct pp2_hif *hif = hifs[core_id];
1136         struct pp2_bpool *bpool = rxq->priv->bpool;
1137
1138         ret = rte_pktmbuf_alloc_bulk(rxq->mp, mbufs, num);
1139         if (ret)
1140                 return ret;
1141
1142         if (cookie_addr_high == MRVL_COOKIE_ADDR_INVALID)
1143                 cookie_addr_high =
1144                         (uint64_t)mbufs[0] & MRVL_COOKIE_HIGH_ADDR_MASK;
1145
1146         for (i = 0; i < num; i++) {
1147                 if (((uint64_t)mbufs[i] & MRVL_COOKIE_HIGH_ADDR_MASK)
1148                         != cookie_addr_high) {
1149                         RTE_LOG(ERR, PMD,
1150                                 "mbuf virtual addr high 0x%lx out of range\n",
1151                                 (uint64_t)mbufs[i] >> 32);
1152                         goto out;
1153                 }
1154
1155                 entries[i].buff.addr =
1156                         rte_mbuf_data_iova_default(mbufs[i]);
1157                 entries[i].buff.cookie = (pp2_cookie_t)(uint64_t)mbufs[i];
1158                 entries[i].bpool = bpool;
1159         }
1160
1161         pp2_bpool_put_buffs(hif, entries, (uint16_t *)&i);
1162         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] += i;
1163
1164         if (i != num)
1165                 goto out;
1166
1167         return 0;
1168 out:
1169         for (; i < num; i++)
1170                 rte_pktmbuf_free(mbufs[i]);
1171
1172         return -1;
1173 }
1174
1175 /**
1176  * DPDK callback to configure the receive queue.
1177  *
1178  * @param dev
1179  *   Pointer to Ethernet device structure.
1180  * @param idx
1181  *   RX queue index.
1182  * @param desc
1183  *   Number of descriptors to configure in queue.
1184  * @param socket
1185  *   NUMA socket on which memory must be allocated.
1186  * @param conf
1187  *   Thresholds parameters (unused_).
1188  * @param mp
1189  *   Memory pool for buffer allocations.
1190  *
1191  * @return
1192  *   0 on success, negative error value otherwise.
1193  */
1194 static int
1195 mrvl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1196                     unsigned int socket,
1197                     const struct rte_eth_rxconf *conf __rte_unused,
1198                     struct rte_mempool *mp)
1199 {
1200         struct mrvl_priv *priv = dev->data->dev_private;
1201         struct mrvl_rxq *rxq;
1202         uint32_t min_size,
1203                  max_rx_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
1204         int ret, tc, inq;
1205
1206         if (priv->rxq_map[idx].tc == MRVL_UNKNOWN_TC) {
1207                 /*
1208                  * Unknown TC mapping, mapping will not have a correct queue.
1209                  */
1210                 RTE_LOG(ERR, PMD, "Unknown TC mapping for queue %hu eth%hhu\n",
1211                         idx, priv->ppio_id);
1212                 return -EFAULT;
1213         }
1214
1215         min_size = rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM -
1216                    MRVL_PKT_EFFEC_OFFS;
1217         if (min_size < max_rx_pkt_len) {
1218                 RTE_LOG(ERR, PMD,
1219                         "Mbuf size must be increased to %u bytes to hold up to %u bytes of data.\n",
1220                         max_rx_pkt_len + RTE_PKTMBUF_HEADROOM +
1221                         MRVL_PKT_EFFEC_OFFS,
1222                         max_rx_pkt_len);
1223                 return -EINVAL;
1224         }
1225
1226         if (dev->data->rx_queues[idx]) {
1227                 rte_free(dev->data->rx_queues[idx]);
1228                 dev->data->rx_queues[idx] = NULL;
1229         }
1230
1231         rxq = rte_zmalloc_socket("rxq", sizeof(*rxq), 0, socket);
1232         if (!rxq)
1233                 return -ENOMEM;
1234
1235         rxq->priv = priv;
1236         rxq->mp = mp;
1237         rxq->cksum_enabled = dev->data->dev_conf.rxmode.hw_ip_checksum;
1238         rxq->queue_id = idx;
1239         rxq->port_id = dev->data->port_id;
1240         mrvl_port_to_bpool_lookup[rxq->port_id] = priv->bpool;
1241
1242         tc = priv->rxq_map[rxq->queue_id].tc,
1243         inq = priv->rxq_map[rxq->queue_id].inq;
1244         priv->ppio_params.inqs_params.tcs_params[tc].inqs_params[inq].size =
1245                 desc;
1246
1247         ret = mrvl_fill_bpool(rxq, desc);
1248         if (ret) {
1249                 rte_free(rxq);
1250                 return ret;
1251         }
1252
1253         priv->bpool_init_size += desc;
1254
1255         dev->data->rx_queues[idx] = rxq;
1256
1257         return 0;
1258 }
1259
1260 /**
1261  * DPDK callback to release the receive queue.
1262  *
1263  * @param rxq
1264  *   Generic receive queue pointer.
1265  */
1266 static void
1267 mrvl_rx_queue_release(void *rxq)
1268 {
1269         struct mrvl_rxq *q = rxq;
1270         struct pp2_ppio_tc_params *tc_params;
1271         int i, num, tc, inq;
1272
1273         if (!q)
1274                 return;
1275
1276         tc = q->priv->rxq_map[q->queue_id].tc;
1277         inq = q->priv->rxq_map[q->queue_id].inq;
1278         tc_params = &q->priv->ppio_params.inqs_params.tcs_params[tc];
1279         num = tc_params->inqs_params[inq].size;
1280         for (i = 0; i < num; i++) {
1281                 struct pp2_buff_inf inf;
1282                 uint64_t addr;
1283
1284                 pp2_bpool_get_buff(hifs[rte_lcore_id()], q->priv->bpool, &inf);
1285                 addr = cookie_addr_high | inf.cookie;
1286                 rte_pktmbuf_free((struct rte_mbuf *)addr);
1287         }
1288
1289         rte_free(q);
1290 }
1291
1292 /**
1293  * DPDK callback to configure the transmit queue.
1294  *
1295  * @param dev
1296  *   Pointer to Ethernet device structure.
1297  * @param idx
1298  *   Transmit queue index.
1299  * @param desc
1300  *   Number of descriptors to configure in the queue.
1301  * @param socket
1302  *   NUMA socket on which memory must be allocated.
1303  * @param conf
1304  *   Thresholds parameters (unused).
1305  *
1306  * @return
1307  *   0 on success, negative error value otherwise.
1308  */
1309 static int
1310 mrvl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1311                     unsigned int socket,
1312                     const struct rte_eth_txconf *conf __rte_unused)
1313 {
1314         struct mrvl_priv *priv = dev->data->dev_private;
1315         struct mrvl_txq *txq;
1316
1317         if (dev->data->tx_queues[idx]) {
1318                 rte_free(dev->data->tx_queues[idx]);
1319                 dev->data->tx_queues[idx] = NULL;
1320         }
1321
1322         txq = rte_zmalloc_socket("txq", sizeof(*txq), 0, socket);
1323         if (!txq)
1324                 return -ENOMEM;
1325
1326         txq->priv = priv;
1327         txq->queue_id = idx;
1328         txq->port_id = dev->data->port_id;
1329         dev->data->tx_queues[idx] = txq;
1330
1331         priv->ppio_params.outqs_params.outqs_params[idx].size = desc;
1332         priv->ppio_params.outqs_params.outqs_params[idx].weight = 1;
1333
1334         return 0;
1335 }
1336
1337 /**
1338  * DPDK callback to release the transmit queue.
1339  *
1340  * @param txq
1341  *   Generic transmit queue pointer.
1342  */
1343 static void
1344 mrvl_tx_queue_release(void *txq)
1345 {
1346         struct mrvl_txq *q = txq;
1347
1348         if (!q)
1349                 return;
1350
1351         rte_free(q);
1352 }
1353
1354 /**
1355  * Update RSS hash configuration
1356  *
1357  * @param dev
1358  *   Pointer to Ethernet device structure.
1359  * @param rss_conf
1360  *   Pointer to RSS configuration.
1361  *
1362  * @return
1363  *   0 on success, negative error value otherwise.
1364  */
1365 static int
1366 mrvl_rss_hash_update(struct rte_eth_dev *dev,
1367                      struct rte_eth_rss_conf *rss_conf)
1368 {
1369         struct mrvl_priv *priv = dev->data->dev_private;
1370
1371         return mrvl_configure_rss(priv, rss_conf);
1372 }
1373
1374 /**
1375  * DPDK callback to get RSS hash configuration.
1376  *
1377  * @param dev
1378  *   Pointer to Ethernet device structure.
1379  * @rss_conf
1380  *   Pointer to RSS configuration.
1381  *
1382  * @return
1383  *   Always 0.
1384  */
1385 static int
1386 mrvl_rss_hash_conf_get(struct rte_eth_dev *dev,
1387                        struct rte_eth_rss_conf *rss_conf)
1388 {
1389         struct mrvl_priv *priv = dev->data->dev_private;
1390         enum pp2_ppio_hash_type hash_type =
1391                 priv->ppio_params.inqs_params.hash_type;
1392
1393         rss_conf->rss_key = NULL;
1394
1395         if (hash_type == PP2_PPIO_HASH_T_NONE)
1396                 rss_conf->rss_hf = 0;
1397         else if (hash_type == PP2_PPIO_HASH_T_2_TUPLE)
1398                 rss_conf->rss_hf = ETH_RSS_IPV4;
1399         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && priv->rss_hf_tcp)
1400                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_TCP;
1401         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && !priv->rss_hf_tcp)
1402                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_UDP;
1403
1404         return 0;
1405 }
1406
1407 static const struct eth_dev_ops mrvl_ops = {
1408         .dev_configure = mrvl_dev_configure,
1409         .dev_start = mrvl_dev_start,
1410         .dev_stop = mrvl_dev_stop,
1411         .dev_set_link_up = mrvl_dev_set_link_up,
1412         .dev_set_link_down = mrvl_dev_set_link_down,
1413         .dev_close = mrvl_dev_close,
1414         .link_update = mrvl_link_update,
1415         .promiscuous_enable = mrvl_promiscuous_enable,
1416         .allmulticast_enable = mrvl_allmulticast_enable,
1417         .promiscuous_disable = mrvl_promiscuous_disable,
1418         .allmulticast_disable = mrvl_allmulticast_disable,
1419         .mac_addr_remove = mrvl_mac_addr_remove,
1420         .mac_addr_add = mrvl_mac_addr_add,
1421         .mac_addr_set = mrvl_mac_addr_set,
1422         .mtu_set = mrvl_mtu_set,
1423         .stats_get = mrvl_stats_get,
1424         .stats_reset = mrvl_stats_reset,
1425         .dev_infos_get = mrvl_dev_infos_get,
1426         .dev_supported_ptypes_get = mrvl_dev_supported_ptypes_get,
1427         .rxq_info_get = mrvl_rxq_info_get,
1428         .txq_info_get = mrvl_txq_info_get,
1429         .vlan_filter_set = mrvl_vlan_filter_set,
1430         .rx_queue_setup = mrvl_rx_queue_setup,
1431         .rx_queue_release = mrvl_rx_queue_release,
1432         .tx_queue_setup = mrvl_tx_queue_setup,
1433         .tx_queue_release = mrvl_tx_queue_release,
1434         .rss_hash_update = mrvl_rss_hash_update,
1435         .rss_hash_conf_get = mrvl_rss_hash_conf_get,
1436 };
1437
1438 /**
1439  * Return packet type information and l3/l4 offsets.
1440  *
1441  * @param desc
1442  *   Pointer to the received packet descriptor.
1443  * @param l3_offset
1444  *   l3 packet offset.
1445  * @param l4_offset
1446  *   l4 packet offset.
1447  *
1448  * @return
1449  *   Packet type information.
1450  */
1451 static inline uint64_t
1452 mrvl_desc_to_packet_type_and_offset(struct pp2_ppio_desc *desc,
1453                                     uint8_t *l3_offset, uint8_t *l4_offset)
1454 {
1455         enum pp2_inq_l3_type l3_type;
1456         enum pp2_inq_l4_type l4_type;
1457         uint64_t packet_type;
1458
1459         pp2_ppio_inq_desc_get_l3_info(desc, &l3_type, l3_offset);
1460         pp2_ppio_inq_desc_get_l4_info(desc, &l4_type, l4_offset);
1461
1462         packet_type = RTE_PTYPE_L2_ETHER;
1463
1464         switch (l3_type) {
1465         case PP2_INQ_L3_TYPE_IPV4_NO_OPTS:
1466                 packet_type |= RTE_PTYPE_L3_IPV4;
1467                 break;
1468         case PP2_INQ_L3_TYPE_IPV4_OK:
1469                 packet_type |= RTE_PTYPE_L3_IPV4_EXT;
1470                 break;
1471         case PP2_INQ_L3_TYPE_IPV4_TTL_ZERO:
1472                 packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
1473                 break;
1474         case PP2_INQ_L3_TYPE_IPV6_NO_EXT:
1475                 packet_type |= RTE_PTYPE_L3_IPV6;
1476                 break;
1477         case PP2_INQ_L3_TYPE_IPV6_EXT:
1478                 packet_type |= RTE_PTYPE_L3_IPV6_EXT;
1479                 break;
1480         case PP2_INQ_L3_TYPE_ARP:
1481                 packet_type |= RTE_PTYPE_L2_ETHER_ARP;
1482                 /*
1483                  * In case of ARP l4_offset is set to wrong value.
1484                  * Set it to proper one so that later on mbuf->l3_len can be
1485                  * calculated subtracting l4_offset and l3_offset.
1486                  */
1487                 *l4_offset = *l3_offset + MRVL_ARP_LENGTH;
1488                 break;
1489         default:
1490                 RTE_LOG(DEBUG, PMD, "Failed to recognise l3 packet type\n");
1491                 break;
1492         }
1493
1494         switch (l4_type) {
1495         case PP2_INQ_L4_TYPE_TCP:
1496                 packet_type |= RTE_PTYPE_L4_TCP;
1497                 break;
1498         case PP2_INQ_L4_TYPE_UDP:
1499                 packet_type |= RTE_PTYPE_L4_UDP;
1500                 break;
1501         default:
1502                 RTE_LOG(DEBUG, PMD, "Failed to recognise l4 packet type\n");
1503                 break;
1504         }
1505
1506         return packet_type;
1507 }
1508
1509 /**
1510  * Get offload information from the received packet descriptor.
1511  *
1512  * @param desc
1513  *   Pointer to the received packet descriptor.
1514  *
1515  * @return
1516  *   Mbuf offload flags.
1517  */
1518 static inline uint64_t
1519 mrvl_desc_to_ol_flags(struct pp2_ppio_desc *desc)
1520 {
1521         uint64_t flags;
1522         enum pp2_inq_desc_status status;
1523
1524         status = pp2_ppio_inq_desc_get_l3_pkt_error(desc);
1525         if (unlikely(status != PP2_DESC_ERR_OK))
1526                 flags = PKT_RX_IP_CKSUM_BAD;
1527         else
1528                 flags = PKT_RX_IP_CKSUM_GOOD;
1529
1530         status = pp2_ppio_inq_desc_get_l4_pkt_error(desc);
1531         if (unlikely(status != PP2_DESC_ERR_OK))
1532                 flags |= PKT_RX_L4_CKSUM_BAD;
1533         else
1534                 flags |= PKT_RX_L4_CKSUM_GOOD;
1535
1536         return flags;
1537 }
1538
1539 /**
1540  * DPDK callback for receive.
1541  *
1542  * @param rxq
1543  *   Generic pointer to the receive queue.
1544  * @param rx_pkts
1545  *   Array to store received packets.
1546  * @param nb_pkts
1547  *   Maximum number of packets in array.
1548  *
1549  * @return
1550  *   Number of packets successfully received.
1551  */
1552 static uint16_t
1553 mrvl_rx_pkt_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
1554 {
1555         struct mrvl_rxq *q = rxq;
1556         struct pp2_ppio_desc descs[nb_pkts];
1557         struct pp2_bpool *bpool;
1558         int i, ret, rx_done = 0;
1559         int num;
1560         unsigned int core_id = rte_lcore_id();
1561
1562         if (unlikely(!q->priv->ppio))
1563                 return 0;
1564
1565         bpool = q->priv->bpool;
1566
1567         ret = pp2_ppio_recv(q->priv->ppio, q->priv->rxq_map[q->queue_id].tc,
1568                             q->priv->rxq_map[q->queue_id].inq, descs, &nb_pkts);
1569         if (unlikely(ret < 0)) {
1570                 RTE_LOG(ERR, PMD, "Failed to receive packets\n");
1571                 return 0;
1572         }
1573         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] -= nb_pkts;
1574
1575         for (i = 0; i < nb_pkts; i++) {
1576                 struct rte_mbuf *mbuf;
1577                 uint8_t l3_offset, l4_offset;
1578                 enum pp2_inq_desc_status status;
1579                 uint64_t addr;
1580
1581                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
1582                         struct pp2_ppio_desc *pref_desc;
1583                         u64 pref_addr;
1584
1585                         pref_desc = &descs[i + MRVL_MUSDK_PREFETCH_SHIFT];
1586                         pref_addr = cookie_addr_high |
1587                                     pp2_ppio_inq_desc_get_cookie(pref_desc);
1588                         rte_mbuf_prefetch_part1((struct rte_mbuf *)(pref_addr));
1589                         rte_mbuf_prefetch_part2((struct rte_mbuf *)(pref_addr));
1590                 }
1591
1592                 addr = cookie_addr_high |
1593                        pp2_ppio_inq_desc_get_cookie(&descs[i]);
1594                 mbuf = (struct rte_mbuf *)addr;
1595                 rte_pktmbuf_reset(mbuf);
1596
1597                 /* drop packet in case of mac, overrun or resource error */
1598                 status = pp2_ppio_inq_desc_get_l2_pkt_error(&descs[i]);
1599                 if (unlikely(status != PP2_DESC_ERR_OK)) {
1600                         struct pp2_buff_inf binf = {
1601                                 .addr = rte_mbuf_data_iova_default(mbuf),
1602                                 .cookie = (pp2_cookie_t)(uint64_t)mbuf,
1603                         };
1604
1605                         pp2_bpool_put_buff(hifs[core_id], bpool, &binf);
1606                         mrvl_port_bpool_size
1607                                 [bpool->pp2_id][bpool->id][core_id]++;
1608                         q->drop_mac++;
1609                         continue;
1610                 }
1611
1612                 mbuf->data_off += MRVL_PKT_EFFEC_OFFS;
1613                 mbuf->pkt_len = pp2_ppio_inq_desc_get_pkt_len(&descs[i]);
1614                 mbuf->data_len = mbuf->pkt_len;
1615                 mbuf->port = q->port_id;
1616                 mbuf->packet_type =
1617                         mrvl_desc_to_packet_type_and_offset(&descs[i],
1618                                                             &l3_offset,
1619                                                             &l4_offset);
1620                 mbuf->l2_len = l3_offset;
1621                 mbuf->l3_len = l4_offset - l3_offset;
1622
1623                 if (likely(q->cksum_enabled))
1624                         mbuf->ol_flags = mrvl_desc_to_ol_flags(&descs[i]);
1625
1626                 rx_pkts[rx_done++] = mbuf;
1627                 q->bytes_recv += mbuf->pkt_len;
1628         }
1629
1630         if (rte_spinlock_trylock(&q->priv->lock) == 1) {
1631                 num = mrvl_get_bpool_size(bpool->pp2_id, bpool->id);
1632
1633                 if (unlikely(num <= q->priv->bpool_min_size ||
1634                              (!rx_done && num < q->priv->bpool_init_size))) {
1635                         ret = mrvl_fill_bpool(q, MRVL_BURST_SIZE);
1636                         if (ret)
1637                                 RTE_LOG(ERR, PMD, "Failed to fill bpool\n");
1638                 } else if (unlikely(num > q->priv->bpool_max_size)) {
1639                         int i;
1640                         int pkt_to_remove = num - q->priv->bpool_init_size;
1641                         struct rte_mbuf *mbuf;
1642                         struct pp2_buff_inf buff;
1643
1644                         RTE_LOG(DEBUG, PMD,
1645                                 "\nport-%d:%d: bpool %d oversize - remove %d buffers (pool size: %d -> %d)\n",
1646                                 bpool->pp2_id, q->priv->ppio->port_id,
1647                                 bpool->id, pkt_to_remove, num,
1648                                 q->priv->bpool_init_size);
1649
1650                         for (i = 0; i < pkt_to_remove; i++) {
1651                                 pp2_bpool_get_buff(hifs[core_id], bpool, &buff);
1652                                 mbuf = (struct rte_mbuf *)
1653                                         (cookie_addr_high | buff.cookie);
1654                                 rte_pktmbuf_free(mbuf);
1655                         }
1656                         mrvl_port_bpool_size
1657                                 [bpool->pp2_id][bpool->id][core_id] -=
1658                                                                 pkt_to_remove;
1659                 }
1660                 rte_spinlock_unlock(&q->priv->lock);
1661         }
1662
1663         return rx_done;
1664 }
1665
1666 /**
1667  * Prepare offload information.
1668  *
1669  * @param ol_flags
1670  *   Offload flags.
1671  * @param packet_type
1672  *   Packet type bitfield.
1673  * @param l3_type
1674  *   Pointer to the pp2_ouq_l3_type structure.
1675  * @param l4_type
1676  *   Pointer to the pp2_outq_l4_type structure.
1677  * @param gen_l3_cksum
1678  *   Will be set to 1 in case l3 checksum is computed.
1679  * @param l4_cksum
1680  *   Will be set to 1 in case l4 checksum is computed.
1681  *
1682  * @return
1683  *   0 on success, negative error value otherwise.
1684  */
1685 static inline int
1686 mrvl_prepare_proto_info(uint64_t ol_flags, uint32_t packet_type,
1687                         enum pp2_outq_l3_type *l3_type,
1688                         enum pp2_outq_l4_type *l4_type,
1689                         int *gen_l3_cksum,
1690                         int *gen_l4_cksum)
1691 {
1692         /*
1693          * Based on ol_flags prepare information
1694          * for pp2_ppio_outq_desc_set_proto_info() which setups descriptor
1695          * for offloading.
1696          */
1697         if (ol_flags & PKT_TX_IPV4) {
1698                 *l3_type = PP2_OUTQ_L3_TYPE_IPV4;
1699                 *gen_l3_cksum = ol_flags & PKT_TX_IP_CKSUM ? 1 : 0;
1700         } else if (ol_flags & PKT_TX_IPV6) {
1701                 *l3_type = PP2_OUTQ_L3_TYPE_IPV6;
1702                 /* no checksum for ipv6 header */
1703                 *gen_l3_cksum = 0;
1704         } else {
1705                 /* if something different then stop processing */
1706                 return -1;
1707         }
1708
1709         ol_flags &= PKT_TX_L4_MASK;
1710         if ((packet_type & RTE_PTYPE_L4_TCP) &&
1711             ol_flags == PKT_TX_TCP_CKSUM) {
1712                 *l4_type = PP2_OUTQ_L4_TYPE_TCP;
1713                 *gen_l4_cksum = 1;
1714         } else if ((packet_type & RTE_PTYPE_L4_UDP) &&
1715                    ol_flags == PKT_TX_UDP_CKSUM) {
1716                 *l4_type = PP2_OUTQ_L4_TYPE_UDP;
1717                 *gen_l4_cksum = 1;
1718         } else {
1719                 *l4_type = PP2_OUTQ_L4_TYPE_OTHER;
1720                 /* no checksum for other type */
1721                 *gen_l4_cksum = 0;
1722         }
1723
1724         return 0;
1725 }
1726
1727 /**
1728  * Release already sent buffers to bpool (buffer-pool).
1729  *
1730  * @param ppio
1731  *   Pointer to the port structure.
1732  * @param hif
1733  *   Pointer to the MUSDK hardware interface.
1734  * @param sq
1735  *   Pointer to the shadow queue.
1736  * @param qid
1737  *   Queue id number.
1738  * @param force
1739  *   Force releasing packets.
1740  */
1741 static inline void
1742 mrvl_free_sent_buffers(struct pp2_ppio *ppio, struct pp2_hif *hif,
1743                        struct mrvl_shadow_txq *sq, int qid, int force)
1744 {
1745         struct buff_release_entry *entry;
1746         uint16_t nb_done = 0, num = 0, skip_bufs = 0;
1747         int i, core_id = rte_lcore_id();
1748
1749         pp2_ppio_get_num_outq_done(ppio, hif, qid, &nb_done);
1750
1751         sq->num_to_release += nb_done;
1752
1753         if (likely(!force &&
1754                    sq->num_to_release < MRVL_PP2_BUF_RELEASE_BURST_SIZE))
1755                 return;
1756
1757         nb_done = sq->num_to_release;
1758         sq->num_to_release = 0;
1759
1760         for (i = 0; i < nb_done; i++) {
1761                 entry = &sq->ent[sq->tail + num];
1762                 if (unlikely(!entry->buff.addr)) {
1763                         RTE_LOG(ERR, PMD,
1764                                 "Shadow memory @%d: cookie(%lx), pa(%lx)!\n",
1765                                 sq->tail, (u64)entry->buff.cookie,
1766                                 (u64)entry->buff.addr);
1767                         skip_bufs = 1;
1768                         goto skip;
1769                 }
1770
1771                 if (unlikely(!entry->bpool)) {
1772                         struct rte_mbuf *mbuf;
1773
1774                         mbuf = (struct rte_mbuf *)
1775                                (cookie_addr_high | entry->buff.cookie);
1776                         rte_pktmbuf_free(mbuf);
1777                         skip_bufs = 1;
1778                         goto skip;
1779                 }
1780
1781                 mrvl_port_bpool_size
1782                         [entry->bpool->pp2_id][entry->bpool->id][core_id]++;
1783                 num++;
1784                 if (unlikely(sq->tail + num == MRVL_PP2_TX_SHADOWQ_SIZE))
1785                         goto skip;
1786                 continue;
1787 skip:
1788                 if (likely(num))
1789                         pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
1790                 num += skip_bufs;
1791                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
1792                 sq->size -= num;
1793                 num = 0;
1794         }
1795
1796         if (likely(num)) {
1797                 pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
1798                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
1799                 sq->size -= num;
1800         }
1801 }
1802
1803 /**
1804  * DPDK callback for transmit.
1805  *
1806  * @param txq
1807  *   Generic pointer transmit queue.
1808  * @param tx_pkts
1809  *   Packets to transmit.
1810  * @param nb_pkts
1811  *   Number of packets in array.
1812  *
1813  * @return
1814  *   Number of packets successfully transmitted.
1815  */
1816 static uint16_t
1817 mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
1818 {
1819         struct mrvl_txq *q = txq;
1820         struct mrvl_shadow_txq *sq = &shadow_txqs[q->port_id][rte_lcore_id()];
1821         struct pp2_hif *hif = hifs[rte_lcore_id()];
1822         struct pp2_ppio_desc descs[nb_pkts];
1823         int i, ret, bytes_sent = 0;
1824         uint16_t num, sq_free_size;
1825         uint64_t addr;
1826
1827         if (unlikely(!q->priv->ppio))
1828                 return 0;
1829
1830         if (sq->size)
1831                 mrvl_free_sent_buffers(q->priv->ppio, hif, sq, q->queue_id, 0);
1832
1833         sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1;
1834         if (unlikely(nb_pkts > sq_free_size)) {
1835                 RTE_LOG(DEBUG, PMD,
1836                         "No room in shadow queue for %d packets! %d packets will be sent.\n",
1837                         nb_pkts, sq_free_size);
1838                 nb_pkts = sq_free_size;
1839         }
1840
1841         for (i = 0; i < nb_pkts; i++) {
1842                 struct rte_mbuf *mbuf = tx_pkts[i];
1843                 int gen_l3_cksum, gen_l4_cksum;
1844                 enum pp2_outq_l3_type l3_type;
1845                 enum pp2_outq_l4_type l4_type;
1846
1847                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
1848                         struct rte_mbuf *pref_pkt_hdr;
1849
1850                         pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT];
1851                         rte_mbuf_prefetch_part1(pref_pkt_hdr);
1852                         rte_mbuf_prefetch_part2(pref_pkt_hdr);
1853                 }
1854
1855                 sq->ent[sq->head].buff.cookie = (pp2_cookie_t)(uint64_t)mbuf;
1856                 sq->ent[sq->head].buff.addr =
1857                         rte_mbuf_data_iova_default(mbuf);
1858                 sq->ent[sq->head].bpool =
1859                         (unlikely(mbuf->port == 0xff || mbuf->refcnt > 1)) ?
1860                          NULL : mrvl_port_to_bpool_lookup[mbuf->port];
1861                 sq->head = (sq->head + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
1862                 sq->size++;
1863
1864                 pp2_ppio_outq_desc_reset(&descs[i]);
1865                 pp2_ppio_outq_desc_set_phys_addr(&descs[i],
1866                                                  rte_pktmbuf_iova(mbuf));
1867                 pp2_ppio_outq_desc_set_pkt_offset(&descs[i], 0);
1868                 pp2_ppio_outq_desc_set_pkt_len(&descs[i],
1869                                                rte_pktmbuf_pkt_len(mbuf));
1870
1871                 bytes_sent += rte_pktmbuf_pkt_len(mbuf);
1872                 /*
1873                  * in case unsupported ol_flags were passed
1874                  * do not update descriptor offload information
1875                  */
1876                 ret = mrvl_prepare_proto_info(mbuf->ol_flags, mbuf->packet_type,
1877                                               &l3_type, &l4_type, &gen_l3_cksum,
1878                                               &gen_l4_cksum);
1879                 if (unlikely(ret))
1880                         continue;
1881
1882                 pp2_ppio_outq_desc_set_proto_info(&descs[i], l3_type, l4_type,
1883                                                   mbuf->l2_len,
1884                                                   mbuf->l2_len + mbuf->l3_len,
1885                                                   gen_l3_cksum, gen_l4_cksum);
1886         }
1887
1888         num = nb_pkts;
1889         pp2_ppio_send(q->priv->ppio, hif, q->queue_id, descs, &nb_pkts);
1890         /* number of packets that were not sent */
1891         if (unlikely(num > nb_pkts)) {
1892                 for (i = nb_pkts; i < num; i++) {
1893                         sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) &
1894                                 MRVL_PP2_TX_SHADOWQ_MASK;
1895                         addr = cookie_addr_high | sq->ent[sq->head].buff.cookie;
1896                         bytes_sent -=
1897                                 rte_pktmbuf_pkt_len((struct rte_mbuf *)addr);
1898                 }
1899                 sq->size -= num - nb_pkts;
1900         }
1901
1902         q->bytes_sent += bytes_sent;
1903
1904         return nb_pkts;
1905 }
1906
1907 /**
1908  * Initialize packet processor.
1909  *
1910  * @return
1911  *   0 on success, negative error value otherwise.
1912  */
1913 static int
1914 mrvl_init_pp2(void)
1915 {
1916         struct pp2_init_params init_params;
1917
1918         memset(&init_params, 0, sizeof(init_params));
1919         init_params.hif_reserved_map = MRVL_MUSDK_HIFS_RESERVED;
1920         init_params.bm_pool_reserved_map = MRVL_MUSDK_BPOOLS_RESERVED;
1921         init_params.rss_tbl_reserved_map = MRVL_MUSDK_RSS_RESERVED;
1922
1923         return pp2_init(&init_params);
1924 }
1925
1926 /**
1927  * Deinitialize packet processor.
1928  *
1929  * @return
1930  *   0 on success, negative error value otherwise.
1931  */
1932 static void
1933 mrvl_deinit_pp2(void)
1934 {
1935         pp2_deinit();
1936 }
1937
1938 /**
1939  * Create private device structure.
1940  *
1941  * @param dev_name
1942  *   Pointer to the port name passed in the initialization parameters.
1943  *
1944  * @return
1945  *   Pointer to the newly allocated private device structure.
1946  */
1947 static struct mrvl_priv *
1948 mrvl_priv_create(const char *dev_name)
1949 {
1950         struct pp2_bpool_params bpool_params;
1951         char match[MRVL_MATCH_LEN];
1952         struct mrvl_priv *priv;
1953         int ret, bpool_bit;
1954
1955         priv = rte_zmalloc_socket(dev_name, sizeof(*priv), 0, rte_socket_id());
1956         if (!priv)
1957                 return NULL;
1958
1959         ret = pp2_netdev_get_ppio_info((char *)(uintptr_t)dev_name,
1960                                        &priv->pp_id, &priv->ppio_id);
1961         if (ret)
1962                 goto out_free_priv;
1963
1964         bpool_bit = mrvl_reserve_bit(&used_bpools[priv->pp_id],
1965                                      PP2_BPOOL_NUM_POOLS);
1966         if (bpool_bit < 0)
1967                 goto out_free_priv;
1968         priv->bpool_bit = bpool_bit;
1969
1970         snprintf(match, sizeof(match), "pool-%d:%d", priv->pp_id,
1971                  priv->bpool_bit);
1972         memset(&bpool_params, 0, sizeof(bpool_params));
1973         bpool_params.match = match;
1974         bpool_params.buff_len = MRVL_PKT_SIZE_MAX + MRVL_PKT_EFFEC_OFFS;
1975         ret = pp2_bpool_init(&bpool_params, &priv->bpool);
1976         if (ret)
1977                 goto out_clear_bpool_bit;
1978
1979         priv->ppio_params.type = PP2_PPIO_T_NIC;
1980         rte_spinlock_init(&priv->lock);
1981
1982         return priv;
1983 out_clear_bpool_bit:
1984         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
1985 out_free_priv:
1986         rte_free(priv);
1987         return NULL;
1988 }
1989
1990 /**
1991  * Create device representing Ethernet port.
1992  *
1993  * @param name
1994  *   Pointer to the port's name.
1995  *
1996  * @return
1997  *   0 on success, negative error value otherwise.
1998  */
1999 static int
2000 mrvl_eth_dev_create(struct rte_vdev_device *vdev, const char *name)
2001 {
2002         int ret, fd = socket(AF_INET, SOCK_DGRAM, 0);
2003         struct rte_eth_dev *eth_dev;
2004         struct mrvl_priv *priv;
2005         struct ifreq req;
2006
2007         eth_dev = rte_eth_dev_allocate(name);
2008         if (!eth_dev)
2009                 return -ENOMEM;
2010
2011         priv = mrvl_priv_create(name);
2012         if (!priv) {
2013                 ret = -ENOMEM;
2014                 goto out_free_dev;
2015         }
2016
2017         eth_dev->data->mac_addrs =
2018                 rte_zmalloc("mac_addrs",
2019                             ETHER_ADDR_LEN * MRVL_MAC_ADDRS_MAX, 0);
2020         if (!eth_dev->data->mac_addrs) {
2021                 RTE_LOG(ERR, PMD, "Failed to allocate space for eth addrs\n");
2022                 ret = -ENOMEM;
2023                 goto out_free_priv;
2024         }
2025
2026         memset(&req, 0, sizeof(req));
2027         strcpy(req.ifr_name, name);
2028         ret = ioctl(fd, SIOCGIFHWADDR, &req);
2029         if (ret)
2030                 goto out_free_mac;
2031
2032         memcpy(eth_dev->data->mac_addrs[0].addr_bytes,
2033                req.ifr_addr.sa_data, ETHER_ADDR_LEN);
2034
2035         eth_dev->rx_pkt_burst = mrvl_rx_pkt_burst;
2036         eth_dev->tx_pkt_burst = mrvl_tx_pkt_burst;
2037         eth_dev->data->dev_private = priv;
2038         eth_dev->device = &vdev->device;
2039         eth_dev->dev_ops = &mrvl_ops;
2040
2041         return 0;
2042 out_free_mac:
2043         rte_free(eth_dev->data->mac_addrs);
2044 out_free_dev:
2045         rte_eth_dev_release_port(eth_dev);
2046 out_free_priv:
2047         rte_free(priv);
2048
2049         return ret;
2050 }
2051
2052 /**
2053  * Cleanup previously created device representing Ethernet port.
2054  *
2055  * @param name
2056  *   Pointer to the port name.
2057  */
2058 static void
2059 mrvl_eth_dev_destroy(const char *name)
2060 {
2061         struct rte_eth_dev *eth_dev;
2062         struct mrvl_priv *priv;
2063
2064         eth_dev = rte_eth_dev_allocated(name);
2065         if (!eth_dev)
2066                 return;
2067
2068         priv = eth_dev->data->dev_private;
2069         pp2_bpool_deinit(priv->bpool);
2070         rte_free(priv);
2071         rte_free(eth_dev->data->mac_addrs);
2072         rte_eth_dev_release_port(eth_dev);
2073 }
2074
2075 /**
2076  * Callback used by rte_kvargs_process() during argument parsing.
2077  *
2078  * @param key
2079  *   Pointer to the parsed key (unused).
2080  * @param value
2081  *   Pointer to the parsed value.
2082  * @param extra_args
2083  *   Pointer to the extra arguments which contains address of the
2084  *   table of pointers to parsed interface names.
2085  *
2086  * @return
2087  *   Always 0.
2088  */
2089 static int
2090 mrvl_get_ifnames(const char *key __rte_unused, const char *value,
2091                  void *extra_args)
2092 {
2093         const char **ifnames = extra_args;
2094
2095         ifnames[mrvl_ports_nb++] = value;
2096
2097         return 0;
2098 }
2099
2100 /**
2101  * Initialize per-lcore MUSDK hardware interfaces (hifs).
2102  *
2103  * @return
2104  *   0 on success, negative error value otherwise.
2105  */
2106 static int
2107 mrvl_init_hifs(void)
2108 {
2109         struct pp2_hif_params params;
2110         char match[MRVL_MATCH_LEN];
2111         int i, ret;
2112
2113         RTE_LCORE_FOREACH(i) {
2114                 ret = mrvl_reserve_bit(&used_hifs, MRVL_MUSDK_HIFS_MAX);
2115                 if (ret < 0)
2116                         return ret;
2117
2118                 snprintf(match, sizeof(match), "hif-%d", ret);
2119                 memset(&params, 0, sizeof(params));
2120                 params.match = match;
2121                 params.out_size = MRVL_PP2_AGGR_TXQD_MAX;
2122                 ret = pp2_hif_init(&params, &hifs[i]);
2123                 if (ret) {
2124                         RTE_LOG(ERR, PMD, "Failed to initialize hif %d\n", i);
2125                         return ret;
2126                 }
2127         }
2128
2129         return 0;
2130 }
2131
2132 /**
2133  * Deinitialize per-lcore MUSDK hardware interfaces (hifs).
2134  */
2135 static void
2136 mrvl_deinit_hifs(void)
2137 {
2138         int i;
2139
2140         RTE_LCORE_FOREACH(i) {
2141                 if (hifs[i])
2142                         pp2_hif_deinit(hifs[i]);
2143         }
2144 }
2145
2146 static void mrvl_set_first_last_cores(int core_id)
2147 {
2148         if (core_id < mrvl_lcore_first)
2149                 mrvl_lcore_first = core_id;
2150
2151         if (core_id > mrvl_lcore_last)
2152                 mrvl_lcore_last = core_id;
2153 }
2154
2155 /**
2156  * DPDK callback to register the virtual device.
2157  *
2158  * @param vdev
2159  *   Pointer to the virtual device.
2160  *
2161  * @return
2162  *   0 on success, negative error value otherwise.
2163  */
2164 static int
2165 rte_pmd_mrvl_probe(struct rte_vdev_device *vdev)
2166 {
2167         struct rte_kvargs *kvlist;
2168         const char *ifnames[PP2_NUM_ETH_PPIO * PP2_NUM_PKT_PROC];
2169         int ret = -EINVAL;
2170         uint32_t i, ifnum, cfgnum, core_id;
2171         const char *params;
2172
2173         params = rte_vdev_device_args(vdev);
2174         if (!params)
2175                 return -EINVAL;
2176
2177         kvlist = rte_kvargs_parse(params, valid_args);
2178         if (!kvlist)
2179                 return -EINVAL;
2180
2181         ifnum = rte_kvargs_count(kvlist, MRVL_IFACE_NAME_ARG);
2182         if (ifnum > RTE_DIM(ifnames))
2183                 goto out_free_kvlist;
2184
2185         rte_kvargs_process(kvlist, MRVL_IFACE_NAME_ARG,
2186                            mrvl_get_ifnames, &ifnames);
2187
2188         cfgnum = rte_kvargs_count(kvlist, MRVL_CFG_ARG);
2189         if (cfgnum > 1) {
2190                 RTE_LOG(ERR, PMD, "Cannot handle more than one config file!\n");
2191                 goto out_free_kvlist;
2192         } else if (cfgnum == 1) {
2193                 rte_kvargs_process(kvlist, MRVL_CFG_ARG,
2194                                    mrvl_get_qoscfg, &mrvl_qos_cfg);
2195         }
2196
2197         /*
2198          * ret == -EEXIST is correct, it means DMA
2199          * has been already initialized (by another PMD).
2200          */
2201         ret = mv_sys_dma_mem_init(MRVL_MUSDK_DMA_MEMSIZE);
2202         if (ret < 0) {
2203                 if (ret != -EEXIST)
2204                         goto out_free_kvlist;
2205                 else
2206                         RTE_LOG(INFO, PMD,
2207                                 "DMA memory has been already initialized by a different driver.\n");
2208         }
2209
2210         ret = mrvl_init_pp2();
2211         if (ret) {
2212                 RTE_LOG(ERR, PMD, "Failed to init PP!\n");
2213                 goto out_deinit_dma;
2214         }
2215
2216         ret = mrvl_init_hifs();
2217         if (ret)
2218                 goto out_deinit_hifs;
2219
2220         for (i = 0; i < ifnum; i++) {
2221                 RTE_LOG(INFO, PMD, "Creating %s\n", ifnames[i]);
2222                 ret = mrvl_eth_dev_create(vdev, ifnames[i]);
2223                 if (ret)
2224                         goto out_cleanup;
2225         }
2226
2227         rte_kvargs_free(kvlist);
2228
2229         memset(mrvl_port_bpool_size, 0, sizeof(mrvl_port_bpool_size));
2230
2231         mrvl_lcore_first = RTE_MAX_LCORE;
2232         mrvl_lcore_last = 0;
2233
2234         RTE_LCORE_FOREACH(core_id) {
2235                 mrvl_set_first_last_cores(core_id);
2236         }
2237
2238         return 0;
2239 out_cleanup:
2240         for (; i > 0; i--)
2241                 mrvl_eth_dev_destroy(ifnames[i]);
2242 out_deinit_hifs:
2243         mrvl_deinit_hifs();
2244         mrvl_deinit_pp2();
2245 out_deinit_dma:
2246         mv_sys_dma_mem_destroy();
2247 out_free_kvlist:
2248         rte_kvargs_free(kvlist);
2249
2250         return ret;
2251 }
2252
2253 /**
2254  * DPDK callback to remove virtual device.
2255  *
2256  * @param vdev
2257  *   Pointer to the removed virtual device.
2258  *
2259  * @return
2260  *   0 on success, negative error value otherwise.
2261  */
2262 static int
2263 rte_pmd_mrvl_remove(struct rte_vdev_device *vdev)
2264 {
2265         int i;
2266         const char *name;
2267
2268         name = rte_vdev_device_name(vdev);
2269         if (!name)
2270                 return -EINVAL;
2271
2272         RTE_LOG(INFO, PMD, "Removing %s\n", name);
2273
2274         for (i = 0; i < rte_eth_dev_count(); i++) {
2275                 char ifname[RTE_ETH_NAME_MAX_LEN];
2276
2277                 rte_eth_dev_get_name_by_port(i, ifname);
2278                 mrvl_eth_dev_destroy(ifname);
2279         }
2280
2281         mrvl_deinit_hifs();
2282         mrvl_deinit_pp2();
2283         mv_sys_dma_mem_destroy();
2284
2285         return 0;
2286 }
2287
2288 static struct rte_vdev_driver pmd_mrvl_drv = {
2289         .probe = rte_pmd_mrvl_probe,
2290         .remove = rte_pmd_mrvl_remove,
2291 };
2292
2293 RTE_PMD_REGISTER_VDEV(net_mrvl, pmd_mrvl_drv);
2294 RTE_PMD_REGISTER_ALIAS(net_mrvl, eth_mrvl);