drivers/net/mvpp2/mrvl_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Marvell International Ltd.
   3  * Copyright(c) 2017 Semihalf.
   4  * All rights reserved.
   5  */
   6
   7 #include <rte_ethdev_driver.h>
   8 #include <rte_kvargs.h>
   9 #include <rte_log.h>
  10 #include <rte_malloc.h>
  11 #include <rte_bus_vdev.h>
  12
  13 /* Unluckily, container_of is defined by both DPDK and MUSDK,
  14  * we'll declare only one version.
  15  *
  16  * Note that it is not used in this PMD anyway.
  17  */
  18 #ifdef container_of
  19 #undef container_of
  20 #endif
  21
  22 #include <fcntl.h>
  23 #include <linux/ethtool.h>
  24 #include <linux/sockios.h>
  25 #include <net/if.h>
  26 #include <net/if_arp.h>
  27 #include <sys/ioctl.h>
  28 #include <sys/socket.h>
  29 #include <sys/stat.h>
  30 #include <sys/types.h>
  31
  32 #include "mrvl_ethdev.h"
  33 #include "mrvl_qos.h"
  34
  35 /* bitmask with reserved hifs */
  36 #define MRVL_MUSDK_HIFS_RESERVED 0x0F
  37 /* bitmask with reserved bpools */
  38 #define MRVL_MUSDK_BPOOLS_RESERVED 0x07
  39 /* bitmask with reserved kernel RSS tables */
  40 #define MRVL_MUSDK_RSS_RESERVED 0x01
  41 /* maximum number of available hifs */
  42 #define MRVL_MUSDK_HIFS_MAX 9
  43
  44 /* prefetch shift */
  45 #define MRVL_MUSDK_PREFETCH_SHIFT 2
  46
  47 /* TCAM has 25 entries reserved for uc/mc filter entries */
  48 #define MRVL_MAC_ADDRS_MAX 25
  49 #define MRVL_MATCH_LEN 16
  50 #define MRVL_PKT_EFFEC_OFFS (MRVL_PKT_OFFS + MV_MH_SIZE)
  51 /* Maximum allowable packet size */
  52 #define MRVL_PKT_SIZE_MAX (10240 - MV_MH_SIZE)
  53
  54 #define MRVL_IFACE_NAME_ARG "iface"
  55 #define MRVL_CFG_ARG "cfg"
  56
  57 #define MRVL_BURST_SIZE 64
  58
  59 #define MRVL_ARP_LENGTH 28
  60
  61 #define MRVL_COOKIE_ADDR_INVALID ~0ULL
  62
  63 #define MRVL_COOKIE_HIGH_ADDR_SHIFT     (sizeof(pp2_cookie_t) * 8)
  64 #define MRVL_COOKIE_HIGH_ADDR_MASK      (~0ULL << MRVL_COOKIE_HIGH_ADDR_SHIFT)
  65
  66 /* Memory size (in bytes) for MUSDK dma buffers */
  67 #define MRVL_MUSDK_DMA_MEMSIZE 41943040
  68
  69 /** Port Rx offload capabilities */
  70 #define MRVL_RX_OFFLOADS (DEV_RX_OFFLOAD_VLAN_FILTER | \
  71                           DEV_RX_OFFLOAD_JUMBO_FRAME | \
  72                           DEV_RX_OFFLOAD_CRC_STRIP | \
  73                           DEV_RX_OFFLOAD_CHECKSUM)
  74
  75 /** Port Tx offloads capabilities */
  76 #define MRVL_TX_OFFLOADS (DEV_TX_OFFLOAD_IPV4_CKSUM | \
  77                           DEV_TX_OFFLOAD_UDP_CKSUM | \
  78                           DEV_TX_OFFLOAD_TCP_CKSUM)
  79
  80 static const char * const valid_args[] = {
  81         MRVL_IFACE_NAME_ARG,
  82         MRVL_CFG_ARG,
  83         NULL
  84 };
  85
  86 static int used_hifs = MRVL_MUSDK_HIFS_RESERVED;
  87 static struct pp2_hif *hifs[RTE_MAX_LCORE];
  88 static int used_bpools[PP2_NUM_PKT_PROC] = {
  89         MRVL_MUSDK_BPOOLS_RESERVED,
  90         MRVL_MUSDK_BPOOLS_RESERVED
  91 };
  92
  93 struct pp2_bpool *mrvl_port_to_bpool_lookup[RTE_MAX_ETHPORTS];
  94 int mrvl_port_bpool_size[PP2_NUM_PKT_PROC][PP2_BPOOL_NUM_POOLS][RTE_MAX_LCORE];
  95 uint64_t cookie_addr_high = MRVL_COOKIE_ADDR_INVALID;
  96
  97 int mrvl_logtype;
  98
  99 struct mrvl_ifnames {
 100         const char *names[PP2_NUM_ETH_PPIO * PP2_NUM_PKT_PROC];
 101         int idx;
 102 };
 103
 104 /*
 105  * To use buffer harvesting based on loopback port shadow queue structure
 106  * was introduced for buffers information bookkeeping.
 107  *
 108  * Before sending the packet, related buffer information (pp2_buff_inf) is
 109  * stored in shadow queue. After packet is transmitted no longer used
 110  * packet buffer is released back to it's original hardware pool,
 111  * on condition it originated from interface.
 112  * In case it  was generated by application itself i.e: mbuf->port field is
 113  * 0xff then its released to software mempool.
 114  */
 115 struct mrvl_shadow_txq {
 116         int head;           /* write index - used when sending buffers */
 117         int tail;           /* read index - used when releasing buffers */
 118         u16 size;           /* queue occupied size */
 119         u16 num_to_release; /* number of buffers sent, that can be released */
 120         struct buff_release_entry ent[MRVL_PP2_TX_SHADOWQ_SIZE]; /* q entries */
 121 };
 122
 123 struct mrvl_rxq {
 124         struct mrvl_priv *priv;
 125         struct rte_mempool *mp;
 126         int queue_id;
 127         int port_id;
 128         int cksum_enabled;
 129         uint64_t bytes_recv;
 130         uint64_t drop_mac;
 131 };
 132
 133 struct mrvl_txq {
 134         struct mrvl_priv *priv;
 135         int queue_id;
 136         int port_id;
 137         uint64_t bytes_sent;
 138         struct mrvl_shadow_txq shadow_txqs[RTE_MAX_LCORE];
 139         int tx_deferred_start;
 140 };
 141
 142 static int mrvl_lcore_first;
 143 static int mrvl_lcore_last;
 144 static int mrvl_dev_num;
 145
 146 static int mrvl_fill_bpool(struct mrvl_rxq *rxq, int num);
 147 static inline void mrvl_free_sent_buffers(struct pp2_ppio *ppio,
 148                         struct pp2_hif *hif, unsigned int core_id,
 149                         struct mrvl_shadow_txq *sq, int qid, int force);
 150
 151 #define MRVL_XSTATS_TBL_ENTRY(name) { \
 152         #name, offsetof(struct pp2_ppio_statistics, name),      \
 153         sizeof(((struct pp2_ppio_statistics *)0)->name)         \
 154 }
 155
 156 /* Table with xstats data */
 157 static struct {
 158         const char *name;
 159         unsigned int offset;
 160         unsigned int size;
 161 } mrvl_xstats_tbl[] = {
 162         MRVL_XSTATS_TBL_ENTRY(rx_bytes),
 163         MRVL_XSTATS_TBL_ENTRY(rx_packets),
 164         MRVL_XSTATS_TBL_ENTRY(rx_unicast_packets),
 165         MRVL_XSTATS_TBL_ENTRY(rx_errors),
 166         MRVL_XSTATS_TBL_ENTRY(rx_fullq_dropped),
 167         MRVL_XSTATS_TBL_ENTRY(rx_bm_dropped),
 168         MRVL_XSTATS_TBL_ENTRY(rx_early_dropped),
 169         MRVL_XSTATS_TBL_ENTRY(rx_fifo_dropped),
 170         MRVL_XSTATS_TBL_ENTRY(rx_cls_dropped),
 171         MRVL_XSTATS_TBL_ENTRY(tx_bytes),
 172         MRVL_XSTATS_TBL_ENTRY(tx_packets),
 173         MRVL_XSTATS_TBL_ENTRY(tx_unicast_packets),
 174         MRVL_XSTATS_TBL_ENTRY(tx_errors)
 175 };
 176
 177 static inline int
 178 mrvl_get_bpool_size(int pp2_id, int pool_id)
 179 {
 180         int i;
 181         int size = 0;
 182
 183         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++)
 184                 size += mrvl_port_bpool_size[pp2_id][pool_id][i];
 185
 186         return size;
 187 }
 188
 189 static inline int
 190 mrvl_reserve_bit(int *bitmap, int max)
 191 {
 192         int n = sizeof(*bitmap) * 8 - __builtin_clz(*bitmap);
 193
 194         if (n >= max)
 195                 return -1;
 196
 197         *bitmap |= 1 << n;
 198
 199         return n;
 200 }
 201
 202 static int
 203 mrvl_init_hif(int core_id)
 204 {
 205         struct pp2_hif_params params;
 206         char match[MRVL_MATCH_LEN];
 207         int ret;
 208
 209         ret = mrvl_reserve_bit(&used_hifs, MRVL_MUSDK_HIFS_MAX);
 210         if (ret < 0) {
 211                 MRVL_LOG(ERR, "Failed to allocate hif %d", core_id);
 212                 return ret;
 213         }
 214
 215         snprintf(match, sizeof(match), "hif-%d", ret);
 216         memset(&params, 0, sizeof(params));
 217         params.match = match;
 218         params.out_size = MRVL_PP2_AGGR_TXQD_MAX;
 219         ret = pp2_hif_init(&params, &hifs[core_id]);
 220         if (ret) {
 221                 MRVL_LOG(ERR, "Failed to initialize hif %d", core_id);
 222                 return ret;
 223         }
 224
 225         return 0;
 226 }
 227
 228 static inline struct pp2_hif*
 229 mrvl_get_hif(struct mrvl_priv *priv, int core_id)
 230 {
 231         int ret;
 232
 233         if (likely(hifs[core_id] != NULL))
 234                 return hifs[core_id];
 235
 236         rte_spinlock_lock(&priv->lock);
 237
 238         ret = mrvl_init_hif(core_id);
 239         if (ret < 0) {
 240                 MRVL_LOG(ERR, "Failed to allocate hif %d", core_id);
 241                 goto out;
 242         }
 243
 244         if (core_id < mrvl_lcore_first)
 245                 mrvl_lcore_first = core_id;
 246
 247         if (core_id > mrvl_lcore_last)
 248                 mrvl_lcore_last = core_id;
 249 out:
 250         rte_spinlock_unlock(&priv->lock);
 251
 252         return hifs[core_id];
 253 }
 254
 255 /**
 256  * Configure rss based on dpdk rss configuration.
 257  *
 258  * @param priv
 259  *   Pointer to private structure.
 260  * @param rss_conf
 261  *   Pointer to RSS configuration.
 262  *
 263  * @return
 264  *   0 on success, negative error value otherwise.
 265  */
 266 static int
 267 mrvl_configure_rss(struct mrvl_priv *priv, struct rte_eth_rss_conf *rss_conf)
 268 {
 269         if (rss_conf->rss_key)
 270                 MRVL_LOG(WARNING, "Changing hash key is not supported");
 271
 272         if (rss_conf->rss_hf == 0) {
 273                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 274         } else if (rss_conf->rss_hf & ETH_RSS_IPV4) {
 275                 priv->ppio_params.inqs_params.hash_type =
 276                         PP2_PPIO_HASH_T_2_TUPLE;
 277         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) {
 278                 priv->ppio_params.inqs_params.hash_type =
 279                         PP2_PPIO_HASH_T_5_TUPLE;
 280                 priv->rss_hf_tcp = 1;
 281         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
 282                 priv->ppio_params.inqs_params.hash_type =
 283                         PP2_PPIO_HASH_T_5_TUPLE;
 284                 priv->rss_hf_tcp = 0;
 285         } else {
 286                 return -EINVAL;
 287         }
 288
 289         return 0;
 290 }
 291
 292 /**
 293  * Ethernet device configuration.
 294  *
 295  * Prepare the driver for a given number of TX and RX queues and
 296  * configure RSS.
 297  *
 298  * @param dev
 299  *   Pointer to Ethernet device structure.
 300  *
 301  * @return
 302  *   0 on success, negative error value otherwise.
 303  */
 304 static int
 305 mrvl_dev_configure(struct rte_eth_dev *dev)
 306 {
 307         struct mrvl_priv *priv = dev->data->dev_private;
 308         int ret;
 309
 310         if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_NONE &&
 311             dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) {
 312                 MRVL_LOG(INFO, "Unsupported rx multi queue mode %d",
 313                         dev->data->dev_conf.rxmode.mq_mode);
 314                 return -EINVAL;
 315         }
 316
 317         /* KEEP_CRC offload flag is not supported by PMD
 318          * can remove the below block when DEV_RX_OFFLOAD_CRC_STRIP removed
 319          */
 320         if (rte_eth_dev_must_keep_crc(dev->data->dev_conf.rxmode.offloads)) {
 321                 MRVL_LOG(INFO, "L2 CRC stripping is always enabled in hw");
 322                 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_CRC_STRIP;
 323         }
 324
 325         if (dev->data->dev_conf.rxmode.split_hdr_size) {
 326                 MRVL_LOG(INFO, "Split headers not supported");
 327                 return -EINVAL;
 328         }
 329
 330         if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME)
 331                 dev->data->mtu = dev->data->dev_conf.rxmode.max_rx_pkt_len -
 332                                  ETHER_HDR_LEN - ETHER_CRC_LEN;
 333
 334         ret = mrvl_configure_rxqs(priv, dev->data->port_id,
 335                                   dev->data->nb_rx_queues);
 336         if (ret < 0)
 337                 return ret;
 338
 339         ret = mrvl_configure_txqs(priv, dev->data->port_id,
 340                                   dev->data->nb_tx_queues);
 341         if (ret < 0)
 342                 return ret;
 343
 344         priv->ppio_params.outqs_params.num_outqs = dev->data->nb_tx_queues;
 345         priv->ppio_params.maintain_stats = 1;
 346         priv->nb_rx_queues = dev->data->nb_rx_queues;
 347
 348         if (dev->data->nb_rx_queues == 1 &&
 349             dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) {
 350                 MRVL_LOG(WARNING, "Disabling hash for 1 rx queue");
 351                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 352
 353                 return 0;
 354         }
 355
 356         return mrvl_configure_rss(priv,
 357                                   &dev->data->dev_conf.rx_adv_conf.rss_conf);
 358 }
 359
 360 /**
 361  * DPDK callback to change the MTU.
 362  *
 363  * Setting the MTU affects hardware MRU (packets larger than the MRU
 364  * will be dropped).
 365  *
 366  * @param dev
 367  *   Pointer to Ethernet device structure.
 368  * @param mtu
 369  *   New MTU.
 370  *
 371  * @return
 372  *   0 on success, negative error value otherwise.
 373  */
 374 static int
 375 mrvl_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 376 {
 377         struct mrvl_priv *priv = dev->data->dev_private;
 378         /* extra MV_MH_SIZE bytes are required for Marvell tag */
 379         uint16_t mru = mtu + MV_MH_SIZE + ETHER_HDR_LEN + ETHER_CRC_LEN;
 380         int ret;
 381
 382         if (mtu < ETHER_MIN_MTU || mru > MRVL_PKT_SIZE_MAX)
 383                 return -EINVAL;
 384
 385         if (!priv->ppio)
 386                 return 0;
 387
 388         ret = pp2_ppio_set_mru(priv->ppio, mru);
 389         if (ret)
 390                 return ret;
 391
 392         return pp2_ppio_set_mtu(priv->ppio, mtu);
 393 }
 394
 395 /**
 396  * DPDK callback to bring the link up.
 397  *
 398  * @param dev
 399  *   Pointer to Ethernet device structure.
 400  *
 401  * @return
 402  *   0 on success, negative error value otherwise.
 403  */
 404 static int
 405 mrvl_dev_set_link_up(struct rte_eth_dev *dev)
 406 {
 407         struct mrvl_priv *priv = dev->data->dev_private;
 408         int ret;
 409
 410         if (!priv->ppio)
 411                 return -EPERM;
 412
 413         ret = pp2_ppio_enable(priv->ppio);
 414         if (ret)
 415                 return ret;
 416
 417         /*
 418          * mtu/mru can be updated if pp2_ppio_enable() was called at least once
 419          * as pp2_ppio_enable() changes port->t_mode from default 0 to
 420          * PP2_TRAFFIC_INGRESS_EGRESS.
 421          *
 422          * Set mtu to default DPDK value here.
 423          */
 424         ret = mrvl_mtu_set(dev, dev->data->mtu);
 425         if (ret)
 426                 pp2_ppio_disable(priv->ppio);
 427
 428         return ret;
 429 }
 430
 431 /**
 432  * DPDK callback to bring the link down.
 433  *
 434  * @param dev
 435  *   Pointer to Ethernet device structure.
 436  *
 437  * @return
 438  *   0 on success, negative error value otherwise.
 439  */
 440 static int
 441 mrvl_dev_set_link_down(struct rte_eth_dev *dev)
 442 {
 443         struct mrvl_priv *priv = dev->data->dev_private;
 444
 445         if (!priv->ppio)
 446                 return -EPERM;
 447
 448         return pp2_ppio_disable(priv->ppio);
 449 }
 450
 451 /**
 452  * DPDK callback to start tx queue.
 453  *
 454  * @param dev
 455  *   Pointer to Ethernet device structure.
 456  * @param queue_id
 457  *   Transmit queue index.
 458  *
 459  * @return
 460  *   0 on success, negative error value otherwise.
 461  */
 462 static int
 463 mrvl_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id)
 464 {
 465         struct mrvl_priv *priv = dev->data->dev_private;
 466         int ret;
 467
 468         if (!priv)
 469                 return -EPERM;
 470
 471         /* passing 1 enables given tx queue */
 472         ret = pp2_ppio_set_outq_state(priv->ppio, queue_id, 1);
 473         if (ret) {
 474                 MRVL_LOG(ERR, "Failed to start txq %d", queue_id);
 475                 return ret;
 476         }
 477
 478         dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 479
 480         return 0;
 481 }
 482
 483 /**
 484  * DPDK callback to stop tx queue.
 485  *
 486  * @param dev
 487  *   Pointer to Ethernet device structure.
 488  * @param queue_id
 489  *   Transmit queue index.
 490  *
 491  * @return
 492  *   0 on success, negative error value otherwise.
 493  */
 494 static int
 495 mrvl_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id)
 496 {
 497         struct mrvl_priv *priv = dev->data->dev_private;
 498         int ret;
 499
 500         if (!priv->ppio)
 501                 return -EPERM;
 502
 503         /* passing 0 disables given tx queue */
 504         ret = pp2_ppio_set_outq_state(priv->ppio, queue_id, 0);
 505         if (ret) {
 506                 MRVL_LOG(ERR, "Failed to stop txq %d", queue_id);
 507                 return ret;
 508         }
 509
 510         dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 511
 512         return 0;
 513 }
 514
 515 /**
 516  * DPDK callback to start the device.
 517  *
 518  * @param dev
 519  *   Pointer to Ethernet device structure.
 520  *
 521  * @return
 522  *   0 on success, negative errno value on failure.
 523  */
 524 static int
 525 mrvl_dev_start(struct rte_eth_dev *dev)
 526 {
 527         struct mrvl_priv *priv = dev->data->dev_private;
 528         char match[MRVL_MATCH_LEN];
 529         int ret = 0, i, def_init_size;
 530
 531         snprintf(match, sizeof(match), "ppio-%d:%d",
 532                  priv->pp_id, priv->ppio_id);
 533         priv->ppio_params.match = match;
 534
 535         /*
 536          * Calculate the minimum bpool size for refill feature as follows:
 537          * 2 default burst sizes multiply by number of rx queues.
 538          * If the bpool size will be below this value, new buffers will
 539          * be added to the pool.
 540          */
 541         priv->bpool_min_size = priv->nb_rx_queues * MRVL_BURST_SIZE * 2;
 542
 543         /* In case initial bpool size configured in queues setup is
 544          * smaller than minimum size add more buffers
 545          */
 546         def_init_size = priv->bpool_min_size + MRVL_BURST_SIZE * 2;
 547         if (priv->bpool_init_size < def_init_size) {
 548                 int buffs_to_add = def_init_size - priv->bpool_init_size;
 549
 550                 priv->bpool_init_size += buffs_to_add;
 551                 ret = mrvl_fill_bpool(dev->data->rx_queues[0], buffs_to_add);
 552                 if (ret)
 553                         MRVL_LOG(ERR, "Failed to add buffers to bpool");
 554         }
 555
 556         /*
 557          * Calculate the maximum bpool size for refill feature as follows:
 558          * maximum number of descriptors in rx queue multiply by number
 559          * of rx queues plus minimum bpool size.
 560          * In case the bpool size will exceed this value, superfluous buffers
 561          * will be removed
 562          */
 563         priv->bpool_max_size = (priv->nb_rx_queues * MRVL_PP2_RXD_MAX) +
 564                                 priv->bpool_min_size;
 565
 566         ret = pp2_ppio_init(&priv->ppio_params, &priv->ppio);
 567         if (ret) {
 568                 MRVL_LOG(ERR, "Failed to init ppio");
 569                 return ret;
 570         }
 571
 572         /*
 573          * In case there are some some stale uc/mc mac addresses flush them
 574          * here. It cannot be done during mrvl_dev_close() as port information
 575          * is already gone at that point (due to pp2_ppio_deinit() in
 576          * mrvl_dev_stop()).
 577          */
 578         if (!priv->uc_mc_flushed) {
 579                 ret = pp2_ppio_flush_mac_addrs(priv->ppio, 1, 1);
 580                 if (ret) {
 581                         MRVL_LOG(ERR,
 582                                 "Failed to flush uc/mc filter list");
 583                         goto out;
 584                 }
 585                 priv->uc_mc_flushed = 1;
 586         }
 587
 588         if (!priv->vlan_flushed) {
 589                 ret = pp2_ppio_flush_vlan(priv->ppio);
 590                 if (ret) {
 591                         MRVL_LOG(ERR, "Failed to flush vlan list");
 592                         /*
 593                          * TODO
 594                          * once pp2_ppio_flush_vlan() is supported jump to out
 595                          * goto out;
 596                          */
 597                 }
 598                 priv->vlan_flushed = 1;
 599         }
 600
 601         /* For default QoS config, don't start classifier. */
 602         if (mrvl_qos_cfg) {
 603                 ret = mrvl_start_qos_mapping(priv);
 604                 if (ret) {
 605                         MRVL_LOG(ERR, "Failed to setup QoS mapping");
 606                         goto out;
 607                 }
 608         }
 609
 610         ret = mrvl_dev_set_link_up(dev);
 611         if (ret) {
 612                 MRVL_LOG(ERR, "Failed to set link up");
 613                 goto out;
 614         }
 615
 616         /* start tx queues */
 617         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 618                 struct mrvl_txq *txq = dev->data->tx_queues[i];
 619
 620                 dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED;
 621
 622                 if (!txq->tx_deferred_start)
 623                         continue;
 624
 625                 /*
 626                  * All txqs are started by default. Stop them
 627                  * so that tx_deferred_start works as expected.
 628                  */
 629                 ret = mrvl_tx_queue_stop(dev, i);
 630                 if (ret)
 631                         goto out;
 632         }
 633
 634         return 0;
 635 out:
 636         MRVL_LOG(ERR, "Failed to start device");
 637         pp2_ppio_deinit(priv->ppio);
 638         return ret;
 639 }
 640
 641 /**
 642  * Flush receive queues.
 643  *
 644  * @param dev
 645  *   Pointer to Ethernet device structure.
 646  */
 647 static void
 648 mrvl_flush_rx_queues(struct rte_eth_dev *dev)
 649 {
 650         int i;
 651
 652         MRVL_LOG(INFO, "Flushing rx queues");
 653         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 654                 int ret, num;
 655
 656                 do {
 657                         struct mrvl_rxq *q = dev->data->rx_queues[i];
 658                         struct pp2_ppio_desc descs[MRVL_PP2_RXD_MAX];
 659
 660                         num = MRVL_PP2_RXD_MAX;
 661                         ret = pp2_ppio_recv(q->priv->ppio,
 662                                             q->priv->rxq_map[q->queue_id].tc,
 663                                             q->priv->rxq_map[q->queue_id].inq,
 664                                             descs, (uint16_t *)&num);
 665                 } while (ret == 0 && num);
 666         }
 667 }
 668
 669 /**
 670  * Flush transmit shadow queues.
 671  *
 672  * @param dev
 673  *   Pointer to Ethernet device structure.
 674  */
 675 static void
 676 mrvl_flush_tx_shadow_queues(struct rte_eth_dev *dev)
 677 {
 678         int i, j;
 679         struct mrvl_txq *txq;
 680
 681         MRVL_LOG(INFO, "Flushing tx shadow queues");
 682         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 683                 txq = (struct mrvl_txq *)dev->data->tx_queues[i];
 684
 685                 for (j = 0; j < RTE_MAX_LCORE; j++) {
 686                         struct mrvl_shadow_txq *sq;
 687
 688                         if (!hifs[j])
 689                                 continue;
 690
 691                         sq = &txq->shadow_txqs[j];
 692                         mrvl_free_sent_buffers(txq->priv->ppio,
 693                                 hifs[j], j, sq, txq->queue_id, 1);
 694                         while (sq->tail != sq->head) {
 695                                 uint64_t addr = cookie_addr_high |
 696                                         sq->ent[sq->tail].buff.cookie;
 697                                 rte_pktmbuf_free(
 698                                         (struct rte_mbuf *)addr);
 699                                 sq->tail = (sq->tail + 1) &
 700                                             MRVL_PP2_TX_SHADOWQ_MASK;
 701                         }
 702                         memset(sq, 0, sizeof(*sq));
 703                 }
 704         }
 705 }
 706
 707 /**
 708  * Flush hardware bpool (buffer-pool).
 709  *
 710  * @param dev
 711  *   Pointer to Ethernet device structure.
 712  */
 713 static void
 714 mrvl_flush_bpool(struct rte_eth_dev *dev)
 715 {
 716         struct mrvl_priv *priv = dev->data->dev_private;
 717         struct pp2_hif *hif;
 718         uint32_t num;
 719         int ret;
 720         unsigned int core_id = rte_lcore_id();
 721
 722         if (core_id == LCORE_ID_ANY)
 723                 core_id = 0;
 724
 725         hif = mrvl_get_hif(priv, core_id);
 726
 727         ret = pp2_bpool_get_num_buffs(priv->bpool, &num);
 728         if (ret) {
 729                 MRVL_LOG(ERR, "Failed to get bpool buffers number");
 730                 return;
 731         }
 732
 733         while (num--) {
 734                 struct pp2_buff_inf inf;
 735                 uint64_t addr;
 736
 737                 ret = pp2_bpool_get_buff(hif, priv->bpool, &inf);
 738                 if (ret)
 739                         break;
 740
 741                 addr = cookie_addr_high | inf.cookie;
 742                 rte_pktmbuf_free((struct rte_mbuf *)addr);
 743         }
 744 }
 745
 746 /**
 747  * DPDK callback to stop the device.
 748  *
 749  * @param dev
 750  *   Pointer to Ethernet device structure.
 751  */
 752 static void
 753 mrvl_dev_stop(struct rte_eth_dev *dev)
 754 {
 755         struct mrvl_priv *priv = dev->data->dev_private;
 756
 757         mrvl_dev_set_link_down(dev);
 758         mrvl_flush_rx_queues(dev);
 759         mrvl_flush_tx_shadow_queues(dev);
 760         if (priv->cls_tbl) {
 761                 pp2_cls_tbl_deinit(priv->cls_tbl);
 762                 priv->cls_tbl = NULL;
 763         }
 764         if (priv->qos_tbl) {
 765                 pp2_cls_qos_tbl_deinit(priv->qos_tbl);
 766                 priv->qos_tbl = NULL;
 767         }
 768         if (priv->ppio)
 769                 pp2_ppio_deinit(priv->ppio);
 770         priv->ppio = NULL;
 771
 772         /* policer must be released after ppio deinitialization */
 773         if (priv->policer) {
 774                 pp2_cls_plcr_deinit(priv->policer);
 775                 priv->policer = NULL;
 776         }
 777 }
 778
 779 /**
 780  * DPDK callback to close the device.
 781  *
 782  * @param dev
 783  *   Pointer to Ethernet device structure.
 784  */
 785 static void
 786 mrvl_dev_close(struct rte_eth_dev *dev)
 787 {
 788         struct mrvl_priv *priv = dev->data->dev_private;
 789         size_t i;
 790
 791         for (i = 0; i < priv->ppio_params.inqs_params.num_tcs; ++i) {
 792                 struct pp2_ppio_tc_params *tc_params =
 793                         &priv->ppio_params.inqs_params.tcs_params[i];
 794
 795                 if (tc_params->inqs_params) {
 796                         rte_free(tc_params->inqs_params);
 797                         tc_params->inqs_params = NULL;
 798                 }
 799         }
 800
 801         mrvl_flush_bpool(dev);
 802 }
 803
 804 /**
 805  * DPDK callback to retrieve physical link information.
 806  *
 807  * @param dev
 808  *   Pointer to Ethernet device structure.
 809  * @param wait_to_complete
 810  *   Wait for request completion (ignored).
 811  *
 812  * @return
 813  *   0 on success, negative error value otherwise.
 814  */
 815 static int
 816 mrvl_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused)
 817 {
 818         /*
 819          * TODO
 820          * once MUSDK provides necessary API use it here
 821          */
 822         struct mrvl_priv *priv = dev->data->dev_private;
 823         struct ethtool_cmd edata;
 824         struct ifreq req;
 825         int ret, fd, link_up;
 826
 827         if (!priv->ppio)
 828                 return -EPERM;
 829
 830         edata.cmd = ETHTOOL_GSET;
 831
 832         strcpy(req.ifr_name, dev->data->name);
 833         req.ifr_data = (void *)&edata;
 834
 835         fd = socket(AF_INET, SOCK_DGRAM, 0);
 836         if (fd == -1)
 837                 return -EFAULT;
 838
 839         ret = ioctl(fd, SIOCETHTOOL, &req);
 840         if (ret == -1) {
 841                 close(fd);
 842                 return -EFAULT;
 843         }
 844
 845         close(fd);
 846
 847         switch (ethtool_cmd_speed(&edata)) {
 848         case SPEED_10:
 849                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10M;
 850                 break;
 851         case SPEED_100:
 852                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_100M;
 853                 break;
 854         case SPEED_1000:
 855                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_1G;
 856                 break;
 857         case SPEED_10000:
 858                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10G;
 859                 break;
 860         default:
 861                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE;
 862         }
 863
 864         dev->data->dev_link.link_duplex = edata.duplex ? ETH_LINK_FULL_DUPLEX :
 865                                                          ETH_LINK_HALF_DUPLEX;
 866         dev->data->dev_link.link_autoneg = edata.autoneg ? ETH_LINK_AUTONEG :
 867                                                            ETH_LINK_FIXED;
 868         pp2_ppio_get_link_state(priv->ppio, &link_up);
 869         dev->data->dev_link.link_status = link_up ? ETH_LINK_UP : ETH_LINK_DOWN;
 870
 871         return 0;
 872 }
 873
 874 /**
 875  * DPDK callback to enable promiscuous mode.
 876  *
 877  * @param dev
 878  *   Pointer to Ethernet device structure.
 879  */
 880 static void
 881 mrvl_promiscuous_enable(struct rte_eth_dev *dev)
 882 {
 883         struct mrvl_priv *priv = dev->data->dev_private;
 884         int ret;
 885
 886         if (!priv->ppio)
 887                 return;
 888
 889         if (priv->isolated)
 890                 return;
 891
 892         ret = pp2_ppio_set_promisc(priv->ppio, 1);
 893         if (ret)
 894                 MRVL_LOG(ERR, "Failed to enable promiscuous mode");
 895 }
 896
 897 /**
 898  * DPDK callback to enable allmulti mode.
 899  *
 900  * @param dev
 901  *   Pointer to Ethernet device structure.
 902  */
 903 static void
 904 mrvl_allmulticast_enable(struct rte_eth_dev *dev)
 905 {
 906         struct mrvl_priv *priv = dev->data->dev_private;
 907         int ret;
 908
 909         if (!priv->ppio)
 910                 return;
 911
 912         if (priv->isolated)
 913                 return;
 914
 915         ret = pp2_ppio_set_mc_promisc(priv->ppio, 1);
 916         if (ret)
 917                 MRVL_LOG(ERR, "Failed enable all-multicast mode");
 918 }
 919
 920 /**
 921  * DPDK callback to disable promiscuous mode.
 922  *
 923  * @param dev
 924  *   Pointer to Ethernet device structure.
 925  */
 926 static void
 927 mrvl_promiscuous_disable(struct rte_eth_dev *dev)
 928 {
 929         struct mrvl_priv *priv = dev->data->dev_private;
 930         int ret;
 931
 932         if (!priv->ppio)
 933                 return;
 934
 935         ret = pp2_ppio_set_promisc(priv->ppio, 0);
 936         if (ret)
 937                 MRVL_LOG(ERR, "Failed to disable promiscuous mode");
 938 }
 939
 940 /**
 941  * DPDK callback to disable allmulticast mode.
 942  *
 943  * @param dev
 944  *   Pointer to Ethernet device structure.
 945  */
 946 static void
 947 mrvl_allmulticast_disable(struct rte_eth_dev *dev)
 948 {
 949         struct mrvl_priv *priv = dev->data->dev_private;
 950         int ret;
 951
 952         if (!priv->ppio)
 953                 return;
 954
 955         ret = pp2_ppio_set_mc_promisc(priv->ppio, 0);
 956         if (ret)
 957                 MRVL_LOG(ERR, "Failed to disable all-multicast mode");
 958 }
 959
 960 /**
 961  * DPDK callback to remove a MAC address.
 962  *
 963  * @param dev
 964  *   Pointer to Ethernet device structure.
 965  * @param index
 966  *   MAC address index.
 967  */
 968 static void
 969 mrvl_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
 970 {
 971         struct mrvl_priv *priv = dev->data->dev_private;
 972         char buf[ETHER_ADDR_FMT_SIZE];
 973         int ret;
 974
 975         if (!priv->ppio)
 976                 return;
 977
 978         if (priv->isolated)
 979                 return;
 980
 981         ret = pp2_ppio_remove_mac_addr(priv->ppio,
 982                                        dev->data->mac_addrs[index].addr_bytes);
 983         if (ret) {
 984                 ether_format_addr(buf, sizeof(buf),
 985                                   &dev->data->mac_addrs[index]);
 986                 MRVL_LOG(ERR, "Failed to remove mac %s", buf);
 987         }
 988 }
 989
 990 /**
 991  * DPDK callback to add a MAC address.
 992  *
 993  * @param dev
 994  *   Pointer to Ethernet device structure.
 995  * @param mac_addr
 996  *   MAC address to register.
 997  * @param index
 998  *   MAC address index.
 999  * @param vmdq
1000  *   VMDq pool index to associate address with (unused).
1001  *
1002  * @return
1003  *   0 on success, negative error value otherwise.
1004  */
1005 static int
1006 mrvl_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
1007                   uint32_t index, uint32_t vmdq __rte_unused)
1008 {
1009         struct mrvl_priv *priv = dev->data->dev_private;
1010         char buf[ETHER_ADDR_FMT_SIZE];
1011         int ret;
1012
1013         if (priv->isolated)
1014                 return -ENOTSUP;
1015
1016         if (index == 0)
1017                 /* For setting index 0, mrvl_mac_addr_set() should be used.*/
1018                 return -1;
1019
1020         if (!priv->ppio)
1021                 return 0;
1022
1023         /*
1024          * Maximum number of uc addresses can be tuned via kernel module mvpp2x
1025          * parameter uc_filter_max. Maximum number of mc addresses is then
1026          * MRVL_MAC_ADDRS_MAX - uc_filter_max. Currently it defaults to 4 and
1027          * 21 respectively.
1028          *
1029          * If more than uc_filter_max uc addresses were added to filter list
1030          * then NIC will switch to promiscuous mode automatically.
1031          *
1032          * If more than MRVL_MAC_ADDRS_MAX - uc_filter_max number mc addresses
1033          * were added to filter list then NIC will switch to all-multicast mode
1034          * automatically.
1035          */
1036         ret = pp2_ppio_add_mac_addr(priv->ppio, mac_addr->addr_bytes);
1037         if (ret) {
1038                 ether_format_addr(buf, sizeof(buf), mac_addr);
1039                 MRVL_LOG(ERR, "Failed to add mac %s", buf);
1040                 return -1;
1041         }
1042
1043         return 0;
1044 }
1045
1046 /**
1047  * DPDK callback to set the primary MAC address.
1048  *
1049  * @param dev
1050  *   Pointer to Ethernet device structure.
1051  * @param mac_addr
1052  *   MAC address to register.
1053  *
1054  * @return
1055  *   0 on success, negative error value otherwise.
1056  */
1057 static int
1058 mrvl_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
1059 {
1060         struct mrvl_priv *priv = dev->data->dev_private;
1061         int ret;
1062
1063         if (!priv->ppio)
1064                 return 0;
1065
1066         if (priv->isolated)
1067                 return -ENOTSUP;
1068
1069         ret = pp2_ppio_set_mac_addr(priv->ppio, mac_addr->addr_bytes);
1070         if (ret) {
1071                 char buf[ETHER_ADDR_FMT_SIZE];
1072                 ether_format_addr(buf, sizeof(buf), mac_addr);
1073                 MRVL_LOG(ERR, "Failed to set mac to %s", buf);
1074         }
1075
1076         return ret;
1077 }
1078
1079 /**
1080  * DPDK callback to get device statistics.
1081  *
1082  * @param dev
1083  *   Pointer to Ethernet device structure.
1084  * @param stats
1085  *   Stats structure output buffer.
1086  *
1087  * @return
1088  *   0 on success, negative error value otherwise.
1089  */
1090 static int
1091 mrvl_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1092 {
1093         struct mrvl_priv *priv = dev->data->dev_private;
1094         struct pp2_ppio_statistics ppio_stats;
1095         uint64_t drop_mac = 0;
1096         unsigned int i, idx, ret;
1097
1098         if (!priv->ppio)
1099                 return -EPERM;
1100
1101         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1102                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
1103                 struct pp2_ppio_inq_statistics rx_stats;
1104
1105                 if (!rxq)
1106                         continue;
1107
1108                 idx = rxq->queue_id;
1109                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
1110                         MRVL_LOG(ERR,
1111                                 "rx queue %d stats out of range (0 - %d)",
1112                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
1113                         continue;
1114                 }
1115
1116                 ret = pp2_ppio_inq_get_statistics(priv->ppio,
1117                                                   priv->rxq_map[idx].tc,
1118                                                   priv->rxq_map[idx].inq,
1119                                                   &rx_stats, 0);
1120                 if (unlikely(ret)) {
1121                         MRVL_LOG(ERR,
1122                                 "Failed to update rx queue %d stats", idx);
1123                         break;
1124                 }
1125
1126                 stats->q_ibytes[idx] = rxq->bytes_recv;
1127                 stats->q_ipackets[idx] = rx_stats.enq_desc - rxq->drop_mac;
1128                 stats->q_errors[idx] = rx_stats.drop_early +
1129                                        rx_stats.drop_fullq +
1130                                        rx_stats.drop_bm +
1131                                        rxq->drop_mac;
1132                 stats->ibytes += rxq->bytes_recv;
1133                 drop_mac += rxq->drop_mac;
1134         }
1135
1136         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1137                 struct mrvl_txq *txq = dev->data->tx_queues[i];
1138                 struct pp2_ppio_outq_statistics tx_stats;
1139
1140                 if (!txq)
1141                         continue;
1142
1143                 idx = txq->queue_id;
1144                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
1145                         MRVL_LOG(ERR,
1146                                 "tx queue %d stats out of range (0 - %d)",
1147                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
1148                 }
1149
1150                 ret = pp2_ppio_outq_get_statistics(priv->ppio, idx,
1151                                                    &tx_stats, 0);
1152                 if (unlikely(ret)) {
1153                         MRVL_LOG(ERR,
1154                                 "Failed to update tx queue %d stats", idx);
1155                         break;
1156                 }
1157
1158                 stats->q_opackets[idx] = tx_stats.deq_desc;
1159                 stats->q_obytes[idx] = txq->bytes_sent;
1160                 stats->obytes += txq->bytes_sent;
1161         }
1162
1163         ret = pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
1164         if (unlikely(ret)) {
1165                 MRVL_LOG(ERR, "Failed to update port statistics");
1166                 return ret;
1167         }
1168
1169         stats->ipackets += ppio_stats.rx_packets - drop_mac;
1170         stats->opackets += ppio_stats.tx_packets;
1171         stats->imissed += ppio_stats.rx_fullq_dropped +
1172                           ppio_stats.rx_bm_dropped +
1173                           ppio_stats.rx_early_dropped +
1174                           ppio_stats.rx_fifo_dropped +
1175                           ppio_stats.rx_cls_dropped;
1176         stats->ierrors = drop_mac;
1177
1178         return 0;
1179 }
1180
1181 /**
1182  * DPDK callback to clear device statistics.
1183  *
1184  * @param dev
1185  *   Pointer to Ethernet device structure.
1186  */
1187 static void
1188 mrvl_stats_reset(struct rte_eth_dev *dev)
1189 {
1190         struct mrvl_priv *priv = dev->data->dev_private;
1191         int i;
1192
1193         if (!priv->ppio)
1194                 return;
1195
1196         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1197                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
1198
1199                 pp2_ppio_inq_get_statistics(priv->ppio, priv->rxq_map[i].tc,
1200                                             priv->rxq_map[i].inq, NULL, 1);
1201                 rxq->bytes_recv = 0;
1202                 rxq->drop_mac = 0;
1203         }
1204
1205         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1206                 struct mrvl_txq *txq = dev->data->tx_queues[i];
1207
1208                 pp2_ppio_outq_get_statistics(priv->ppio, i, NULL, 1);
1209                 txq->bytes_sent = 0;
1210         }
1211
1212         pp2_ppio_get_statistics(priv->ppio, NULL, 1);
1213 }
1214
1215 /**
1216  * DPDK callback to get extended statistics.
1217  *
1218  * @param dev
1219  *   Pointer to Ethernet device structure.
1220  * @param stats
1221  *   Pointer to xstats table.
1222  * @param n
1223  *   Number of entries in xstats table.
1224  * @return
1225  *   Negative value on error, number of read xstats otherwise.
1226  */
1227 static int
1228 mrvl_xstats_get(struct rte_eth_dev *dev,
1229                 struct rte_eth_xstat *stats, unsigned int n)
1230 {
1231         struct mrvl_priv *priv = dev->data->dev_private;
1232         struct pp2_ppio_statistics ppio_stats;
1233         unsigned int i;
1234
1235         if (!stats)
1236                 return 0;
1237
1238         pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
1239         for (i = 0; i < n && i < RTE_DIM(mrvl_xstats_tbl); i++) {
1240                 uint64_t val;
1241
1242                 if (mrvl_xstats_tbl[i].size == sizeof(uint32_t))
1243                         val = *(uint32_t *)((uint8_t *)&ppio_stats +
1244                                             mrvl_xstats_tbl[i].offset);
1245                 else if (mrvl_xstats_tbl[i].size == sizeof(uint64_t))
1246                         val = *(uint64_t *)((uint8_t *)&ppio_stats +
1247                                             mrvl_xstats_tbl[i].offset);
1248                 else
1249                         return -EINVAL;
1250
1251                 stats[i].id = i;
1252                 stats[i].value = val;
1253         }
1254
1255         return n;
1256 }
1257
1258 /**
1259  * DPDK callback to reset extended statistics.
1260  *
1261  * @param dev
1262  *   Pointer to Ethernet device structure.
1263  */
1264 static void
1265 mrvl_xstats_reset(struct rte_eth_dev *dev)
1266 {
1267         mrvl_stats_reset(dev);
1268 }
1269
1270 /**
1271  * DPDK callback to get extended statistics names.
1272  *
1273  * @param dev (unused)
1274  *   Pointer to Ethernet device structure.
1275  * @param xstats_names
1276  *   Pointer to xstats names table.
1277  * @param size
1278  *   Size of the xstats names table.
1279  * @return
1280  *   Number of read names.
1281  */
1282 static int
1283 mrvl_xstats_get_names(struct rte_eth_dev *dev __rte_unused,
1284                       struct rte_eth_xstat_name *xstats_names,
1285                       unsigned int size)
1286 {
1287         unsigned int i;
1288
1289         if (!xstats_names)
1290                 return RTE_DIM(mrvl_xstats_tbl);
1291
1292         for (i = 0; i < size && i < RTE_DIM(mrvl_xstats_tbl); i++)
1293                 snprintf(xstats_names[i].name, RTE_ETH_XSTATS_NAME_SIZE, "%s",
1294                          mrvl_xstats_tbl[i].name);
1295
1296         return size;
1297 }
1298
1299 /**
1300  * DPDK callback to get information about the device.
1301  *
1302  * @param dev
1303  *   Pointer to Ethernet device structure (unused).
1304  * @param info
1305  *   Info structure output buffer.
1306  */
1307 static void
1308 mrvl_dev_infos_get(struct rte_eth_dev *dev __rte_unused,
1309                    struct rte_eth_dev_info *info)
1310 {
1311         info->speed_capa = ETH_LINK_SPEED_10M |
1312                            ETH_LINK_SPEED_100M |
1313                            ETH_LINK_SPEED_1G |
1314                            ETH_LINK_SPEED_10G;
1315
1316         info->max_rx_queues = MRVL_PP2_RXQ_MAX;
1317         info->max_tx_queues = MRVL_PP2_TXQ_MAX;
1318         info->max_mac_addrs = MRVL_MAC_ADDRS_MAX;
1319
1320         info->rx_desc_lim.nb_max = MRVL_PP2_RXD_MAX;
1321         info->rx_desc_lim.nb_min = MRVL_PP2_RXD_MIN;
1322         info->rx_desc_lim.nb_align = MRVL_PP2_RXD_ALIGN;
1323
1324         info->tx_desc_lim.nb_max = MRVL_PP2_TXD_MAX;
1325         info->tx_desc_lim.nb_min = MRVL_PP2_TXD_MIN;
1326         info->tx_desc_lim.nb_align = MRVL_PP2_TXD_ALIGN;
1327
1328         info->rx_offload_capa = MRVL_RX_OFFLOADS;
1329         info->rx_queue_offload_capa = MRVL_RX_OFFLOADS;
1330
1331         info->tx_offload_capa = MRVL_TX_OFFLOADS;
1332         info->tx_queue_offload_capa = MRVL_TX_OFFLOADS;
1333
1334         info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1335                                        ETH_RSS_NONFRAG_IPV4_TCP |
1336                                        ETH_RSS_NONFRAG_IPV4_UDP;
1337
1338         /* By default packets are dropped if no descriptors are available */
1339         info->default_rxconf.rx_drop_en = 1;
1340         info->default_rxconf.offloads = DEV_RX_OFFLOAD_CRC_STRIP;
1341
1342         info->max_rx_pktlen = MRVL_PKT_SIZE_MAX;
1343 }
1344
1345 /**
1346  * Return supported packet types.
1347  *
1348  * @param dev
1349  *   Pointer to Ethernet device structure (unused).
1350  *
1351  * @return
1352  *   Const pointer to the table with supported packet types.
1353  */
1354 static const uint32_t *
1355 mrvl_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1356 {
1357         static const uint32_t ptypes[] = {
1358                 RTE_PTYPE_L2_ETHER,
1359                 RTE_PTYPE_L3_IPV4,
1360                 RTE_PTYPE_L3_IPV4_EXT,
1361                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
1362                 RTE_PTYPE_L3_IPV6,
1363                 RTE_PTYPE_L3_IPV6_EXT,
1364                 RTE_PTYPE_L2_ETHER_ARP,
1365                 RTE_PTYPE_L4_TCP,
1366                 RTE_PTYPE_L4_UDP
1367         };
1368
1369         return ptypes;
1370 }
1371
1372 /**
1373  * DPDK callback to get information about specific receive queue.
1374  *
1375  * @param dev
1376  *   Pointer to Ethernet device structure.
1377  * @param rx_queue_id
1378  *   Receive queue index.
1379  * @param qinfo
1380  *   Receive queue information structure.
1381  */
1382 static void mrvl_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
1383                               struct rte_eth_rxq_info *qinfo)
1384 {
1385         struct mrvl_rxq *q = dev->data->rx_queues[rx_queue_id];
1386         struct mrvl_priv *priv = dev->data->dev_private;
1387         int inq = priv->rxq_map[rx_queue_id].inq;
1388         int tc = priv->rxq_map[rx_queue_id].tc;
1389         struct pp2_ppio_tc_params *tc_params =
1390                 &priv->ppio_params.inqs_params.tcs_params[tc];
1391
1392         qinfo->mp = q->mp;
1393         qinfo->nb_desc = tc_params->inqs_params[inq].size;
1394 }
1395
1396 /**
1397  * DPDK callback to get information about specific transmit queue.
1398  *
1399  * @param dev
1400  *   Pointer to Ethernet device structure.
1401  * @param tx_queue_id
1402  *   Transmit queue index.
1403  * @param qinfo
1404  *   Transmit queue information structure.
1405  */
1406 static void mrvl_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
1407                               struct rte_eth_txq_info *qinfo)
1408 {
1409         struct mrvl_priv *priv = dev->data->dev_private;
1410         struct mrvl_txq *txq = dev->data->tx_queues[tx_queue_id];
1411
1412         qinfo->nb_desc =
1413                 priv->ppio_params.outqs_params.outqs_params[tx_queue_id].size;
1414         qinfo->conf.tx_deferred_start = txq->tx_deferred_start;
1415 }
1416
1417 /**
1418  * DPDK callback to Configure a VLAN filter.
1419  *
1420  * @param dev
1421  *   Pointer to Ethernet device structure.
1422  * @param vlan_id
1423  *   VLAN ID to filter.
1424  * @param on
1425  *   Toggle filter.
1426  *
1427  * @return
1428  *   0 on success, negative error value otherwise.
1429  */
1430 static int
1431 mrvl_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1432 {
1433         struct mrvl_priv *priv = dev->data->dev_private;
1434
1435         if (!priv->ppio)
1436                 return -EPERM;
1437
1438         if (priv->isolated)
1439                 return -ENOTSUP;
1440
1441         return on ? pp2_ppio_add_vlan(priv->ppio, vlan_id) :
1442                     pp2_ppio_remove_vlan(priv->ppio, vlan_id);
1443 }
1444
1445 /**
1446  * Release buffers to hardware bpool (buffer-pool)
1447  *
1448  * @param rxq
1449  *   Receive queue pointer.
1450  * @param num
1451  *   Number of buffers to release to bpool.
1452  *
1453  * @return
1454  *   0 on success, negative error value otherwise.
1455  */
1456 static int
1457 mrvl_fill_bpool(struct mrvl_rxq *rxq, int num)
1458 {
1459         struct buff_release_entry entries[MRVL_PP2_RXD_MAX];
1460         struct rte_mbuf *mbufs[MRVL_PP2_RXD_MAX];
1461         int i, ret;
1462         unsigned int core_id;
1463         struct pp2_hif *hif;
1464         struct pp2_bpool *bpool;
1465
1466         core_id = rte_lcore_id();
1467         if (core_id == LCORE_ID_ANY)
1468                 core_id = 0;
1469
1470         hif = mrvl_get_hif(rxq->priv, core_id);
1471         if (!hif)
1472                 return -1;
1473
1474         bpool = rxq->priv->bpool;
1475
1476         ret = rte_pktmbuf_alloc_bulk(rxq->mp, mbufs, num);
1477         if (ret)
1478                 return ret;
1479
1480         if (cookie_addr_high == MRVL_COOKIE_ADDR_INVALID)
1481                 cookie_addr_high =
1482                         (uint64_t)mbufs[0] & MRVL_COOKIE_HIGH_ADDR_MASK;
1483
1484         for (i = 0; i < num; i++) {
1485                 if (((uint64_t)mbufs[i] & MRVL_COOKIE_HIGH_ADDR_MASK)
1486                         != cookie_addr_high) {
1487                         MRVL_LOG(ERR,
1488                                 "mbuf virtual addr high 0x%lx out of range",
1489                                 (uint64_t)mbufs[i] >> 32);
1490                         goto out;
1491                 }
1492
1493                 entries[i].buff.addr =
1494                         rte_mbuf_data_iova_default(mbufs[i]);
1495                 entries[i].buff.cookie = (pp2_cookie_t)(uint64_t)mbufs[i];
1496                 entries[i].bpool = bpool;
1497         }
1498
1499         pp2_bpool_put_buffs(hif, entries, (uint16_t *)&i);
1500         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] += i;
1501
1502         if (i != num)
1503                 goto out;
1504
1505         return 0;
1506 out:
1507         for (; i < num; i++)
1508                 rte_pktmbuf_free(mbufs[i]);
1509
1510         return -1;
1511 }
1512
1513 /**
1514  * DPDK callback to configure the receive queue.
1515  *
1516  * @param dev
1517  *   Pointer to Ethernet device structure.
1518  * @param idx
1519  *   RX queue index.
1520  * @param desc
1521  *   Number of descriptors to configure in queue.
1522  * @param socket
1523  *   NUMA socket on which memory must be allocated.
1524  * @param conf
1525  *   Thresholds parameters.
1526  * @param mp
1527  *   Memory pool for buffer allocations.
1528  *
1529  * @return
1530  *   0 on success, negative error value otherwise.
1531  */
1532 static int
1533 mrvl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1534                     unsigned int socket,
1535                     const struct rte_eth_rxconf *conf,
1536                     struct rte_mempool *mp)
1537 {
1538         struct mrvl_priv *priv = dev->data->dev_private;
1539         struct mrvl_rxq *rxq;
1540         uint32_t min_size,
1541                  max_rx_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
1542         int ret, tc, inq;
1543         uint64_t offloads;
1544
1545         offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads;
1546
1547         if (priv->rxq_map[idx].tc == MRVL_UNKNOWN_TC) {
1548                 /*
1549                  * Unknown TC mapping, mapping will not have a correct queue.
1550                  */
1551                 MRVL_LOG(ERR, "Unknown TC mapping for queue %hu eth%hhu",
1552                         idx, priv->ppio_id);
1553                 return -EFAULT;
1554         }
1555
1556         min_size = rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM -
1557                    MRVL_PKT_EFFEC_OFFS;
1558         if (min_size < max_rx_pkt_len) {
1559                 MRVL_LOG(ERR,
1560                         "Mbuf size must be increased to %u bytes to hold up to %u bytes of data.",
1561                         max_rx_pkt_len + RTE_PKTMBUF_HEADROOM +
1562                         MRVL_PKT_EFFEC_OFFS,
1563                         max_rx_pkt_len);
1564                 return -EINVAL;
1565         }
1566
1567         if (dev->data->rx_queues[idx]) {
1568                 rte_free(dev->data->rx_queues[idx]);
1569                 dev->data->rx_queues[idx] = NULL;
1570         }
1571
1572         rxq = rte_zmalloc_socket("rxq", sizeof(*rxq), 0, socket);
1573         if (!rxq)
1574                 return -ENOMEM;
1575
1576         rxq->priv = priv;
1577         rxq->mp = mp;
1578         rxq->cksum_enabled = offloads & DEV_RX_OFFLOAD_IPV4_CKSUM;
1579         rxq->queue_id = idx;
1580         rxq->port_id = dev->data->port_id;
1581         mrvl_port_to_bpool_lookup[rxq->port_id] = priv->bpool;
1582
1583         tc = priv->rxq_map[rxq->queue_id].tc,
1584         inq = priv->rxq_map[rxq->queue_id].inq;
1585         priv->ppio_params.inqs_params.tcs_params[tc].inqs_params[inq].size =
1586                 desc;
1587
1588         ret = mrvl_fill_bpool(rxq, desc);
1589         if (ret) {
1590                 rte_free(rxq);
1591                 return ret;
1592         }
1593
1594         priv->bpool_init_size += desc;
1595
1596         dev->data->rx_queues[idx] = rxq;
1597
1598         return 0;
1599 }
1600
1601 /**
1602  * DPDK callback to release the receive queue.
1603  *
1604  * @param rxq
1605  *   Generic receive queue pointer.
1606  */
1607 static void
1608 mrvl_rx_queue_release(void *rxq)
1609 {
1610         struct mrvl_rxq *q = rxq;
1611         struct pp2_ppio_tc_params *tc_params;
1612         int i, num, tc, inq;
1613         struct pp2_hif *hif;
1614         unsigned int core_id = rte_lcore_id();
1615
1616         if (core_id == LCORE_ID_ANY)
1617                 core_id = 0;
1618
1619         if (!q)
1620                 return;
1621
1622         hif = mrvl_get_hif(q->priv, core_id);
1623
1624         if (!hif)
1625                 return;
1626
1627         tc = q->priv->rxq_map[q->queue_id].tc;
1628         inq = q->priv->rxq_map[q->queue_id].inq;
1629         tc_params = &q->priv->ppio_params.inqs_params.tcs_params[tc];
1630         num = tc_params->inqs_params[inq].size;
1631         for (i = 0; i < num; i++) {
1632                 struct pp2_buff_inf inf;
1633                 uint64_t addr;
1634
1635                 pp2_bpool_get_buff(hif, q->priv->bpool, &inf);
1636                 addr = cookie_addr_high | inf.cookie;
1637                 rte_pktmbuf_free((struct rte_mbuf *)addr);
1638         }
1639
1640         rte_free(q);
1641 }
1642
1643 /**
1644  * DPDK callback to configure the transmit queue.
1645  *
1646  * @param dev
1647  *   Pointer to Ethernet device structure.
1648  * @param idx
1649  *   Transmit queue index.
1650  * @param desc
1651  *   Number of descriptors to configure in the queue.
1652  * @param socket
1653  *   NUMA socket on which memory must be allocated.
1654  * @param conf
1655  *   Tx queue configuration parameters.
1656  *
1657  * @return
1658  *   0 on success, negative error value otherwise.
1659  */
1660 static int
1661 mrvl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1662                     unsigned int socket,
1663                     const struct rte_eth_txconf *conf)
1664 {
1665         struct mrvl_priv *priv = dev->data->dev_private;
1666         struct mrvl_txq *txq;
1667
1668         if (dev->data->tx_queues[idx]) {
1669                 rte_free(dev->data->tx_queues[idx]);
1670                 dev->data->tx_queues[idx] = NULL;
1671         }
1672
1673         txq = rte_zmalloc_socket("txq", sizeof(*txq), 0, socket);
1674         if (!txq)
1675                 return -ENOMEM;
1676
1677         txq->priv = priv;
1678         txq->queue_id = idx;
1679         txq->port_id = dev->data->port_id;
1680         txq->tx_deferred_start = conf->tx_deferred_start;
1681         dev->data->tx_queues[idx] = txq;
1682
1683         priv->ppio_params.outqs_params.outqs_params[idx].size = desc;
1684
1685         return 0;
1686 }
1687
1688 /**
1689  * DPDK callback to release the transmit queue.
1690  *
1691  * @param txq
1692  *   Generic transmit queue pointer.
1693  */
1694 static void
1695 mrvl_tx_queue_release(void *txq)
1696 {
1697         struct mrvl_txq *q = txq;
1698
1699         if (!q)
1700                 return;
1701
1702         rte_free(q);
1703 }
1704
1705 /**
1706  * DPDK callback to get flow control configuration.
1707  *
1708  * @param dev
1709  *  Pointer to Ethernet device structure.
1710  * @param fc_conf
1711  *  Pointer to the flow control configuration.
1712  *
1713  * @return
1714  *  0 on success, negative error value otherwise.
1715  */
1716 static int
1717 mrvl_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1718 {
1719         struct mrvl_priv *priv = dev->data->dev_private;
1720         int ret, en;
1721
1722         if (!priv)
1723                 return -EPERM;
1724
1725         ret = pp2_ppio_get_rx_pause(priv->ppio, &en);
1726         if (ret) {
1727                 MRVL_LOG(ERR, "Failed to read rx pause state");
1728                 return ret;
1729         }
1730
1731         fc_conf->mode = en ? RTE_FC_RX_PAUSE : RTE_FC_NONE;
1732
1733         return 0;
1734 }
1735
1736 /**
1737  * DPDK callback to set flow control configuration.
1738  *
1739  * @param dev
1740  *  Pointer to Ethernet device structure.
1741  * @param fc_conf
1742  *  Pointer to the flow control configuration.
1743  *
1744  * @return
1745  *  0 on success, negative error value otherwise.
1746  */
1747 static int
1748 mrvl_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1749 {
1750         struct mrvl_priv *priv = dev->data->dev_private;
1751
1752         if (!priv)
1753                 return -EPERM;
1754
1755         if (fc_conf->high_water ||
1756             fc_conf->low_water ||
1757             fc_conf->pause_time ||
1758             fc_conf->mac_ctrl_frame_fwd ||
1759             fc_conf->autoneg) {
1760                 MRVL_LOG(ERR, "Flowctrl parameter is not supported");
1761
1762                 return -EINVAL;
1763         }
1764
1765         if (fc_conf->mode == RTE_FC_NONE ||
1766             fc_conf->mode == RTE_FC_RX_PAUSE) {
1767                 int ret, en;
1768
1769                 en = fc_conf->mode == RTE_FC_NONE ? 0 : 1;
1770                 ret = pp2_ppio_set_rx_pause(priv->ppio, en);
1771                 if (ret)
1772                         MRVL_LOG(ERR,
1773                                 "Failed to change flowctrl on RX side");
1774
1775                 return ret;
1776         }
1777
1778         return 0;
1779 }
1780
1781 /**
1782  * Update RSS hash configuration
1783  *
1784  * @param dev
1785  *   Pointer to Ethernet device structure.
1786  * @param rss_conf
1787  *   Pointer to RSS configuration.
1788  *
1789  * @return
1790  *   0 on success, negative error value otherwise.
1791  */
1792 static int
1793 mrvl_rss_hash_update(struct rte_eth_dev *dev,
1794                      struct rte_eth_rss_conf *rss_conf)
1795 {
1796         struct mrvl_priv *priv = dev->data->dev_private;
1797
1798         if (priv->isolated)
1799                 return -ENOTSUP;
1800
1801         return mrvl_configure_rss(priv, rss_conf);
1802 }
1803
1804 /**
1805  * DPDK callback to get RSS hash configuration.
1806  *
1807  * @param dev
1808  *   Pointer to Ethernet device structure.
1809  * @rss_conf
1810  *   Pointer to RSS configuration.
1811  *
1812  * @return
1813  *   Always 0.
1814  */
1815 static int
1816 mrvl_rss_hash_conf_get(struct rte_eth_dev *dev,
1817                        struct rte_eth_rss_conf *rss_conf)
1818 {
1819         struct mrvl_priv *priv = dev->data->dev_private;
1820         enum pp2_ppio_hash_type hash_type =
1821                 priv->ppio_params.inqs_params.hash_type;
1822
1823         rss_conf->rss_key = NULL;
1824
1825         if (hash_type == PP2_PPIO_HASH_T_NONE)
1826                 rss_conf->rss_hf = 0;
1827         else if (hash_type == PP2_PPIO_HASH_T_2_TUPLE)
1828                 rss_conf->rss_hf = ETH_RSS_IPV4;
1829         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && priv->rss_hf_tcp)
1830                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_TCP;
1831         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && !priv->rss_hf_tcp)
1832                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_UDP;
1833
1834         return 0;
1835 }
1836
1837 /**
1838  * DPDK callback to get rte_flow callbacks.
1839  *
1840  * @param dev
1841  *   Pointer to the device structure.
1842  * @param filer_type
1843  *   Flow filter type.
1844  * @param filter_op
1845  *   Flow filter operation.
1846  * @param arg
1847  *   Pointer to pass the flow ops.
1848  *
1849  * @return
1850  *   0 on success, negative error value otherwise.
1851  */
1852 static int
1853 mrvl_eth_filter_ctrl(struct rte_eth_dev *dev __rte_unused,
1854                      enum rte_filter_type filter_type,
1855                      enum rte_filter_op filter_op, void *arg)
1856 {
1857         switch (filter_type) {
1858         case RTE_ETH_FILTER_GENERIC:
1859                 if (filter_op != RTE_ETH_FILTER_GET)
1860                         return -EINVAL;
1861                 *(const void **)arg = &mrvl_flow_ops;
1862                 return 0;
1863         default:
1864                 MRVL_LOG(WARNING, "Filter type (%d) not supported",
1865                                 filter_type);
1866                 return -EINVAL;
1867         }
1868 }
1869
1870 static const struct eth_dev_ops mrvl_ops = {
1871         .dev_configure = mrvl_dev_configure,
1872         .dev_start = mrvl_dev_start,
1873         .dev_stop = mrvl_dev_stop,
1874         .dev_set_link_up = mrvl_dev_set_link_up,
1875         .dev_set_link_down = mrvl_dev_set_link_down,
1876         .dev_close = mrvl_dev_close,
1877         .link_update = mrvl_link_update,
1878         .promiscuous_enable = mrvl_promiscuous_enable,
1879         .allmulticast_enable = mrvl_allmulticast_enable,
1880         .promiscuous_disable = mrvl_promiscuous_disable,
1881         .allmulticast_disable = mrvl_allmulticast_disable,
1882         .mac_addr_remove = mrvl_mac_addr_remove,
1883         .mac_addr_add = mrvl_mac_addr_add,
1884         .mac_addr_set = mrvl_mac_addr_set,
1885         .mtu_set = mrvl_mtu_set,
1886         .stats_get = mrvl_stats_get,
1887         .stats_reset = mrvl_stats_reset,
1888         .xstats_get = mrvl_xstats_get,
1889         .xstats_reset = mrvl_xstats_reset,
1890         .xstats_get_names = mrvl_xstats_get_names,
1891         .dev_infos_get = mrvl_dev_infos_get,
1892         .dev_supported_ptypes_get = mrvl_dev_supported_ptypes_get,
1893         .rxq_info_get = mrvl_rxq_info_get,
1894         .txq_info_get = mrvl_txq_info_get,
1895         .vlan_filter_set = mrvl_vlan_filter_set,
1896         .tx_queue_start = mrvl_tx_queue_start,
1897         .tx_queue_stop = mrvl_tx_queue_stop,
1898         .rx_queue_setup = mrvl_rx_queue_setup,
1899         .rx_queue_release = mrvl_rx_queue_release,
1900         .tx_queue_setup = mrvl_tx_queue_setup,
1901         .tx_queue_release = mrvl_tx_queue_release,
1902         .flow_ctrl_get = mrvl_flow_ctrl_get,
1903         .flow_ctrl_set = mrvl_flow_ctrl_set,
1904         .rss_hash_update = mrvl_rss_hash_update,
1905         .rss_hash_conf_get = mrvl_rss_hash_conf_get,
1906         .filter_ctrl = mrvl_eth_filter_ctrl,
1907 };
1908
1909 /**
1910  * Return packet type information and l3/l4 offsets.
1911  *
1912  * @param desc
1913  *   Pointer to the received packet descriptor.
1914  * @param l3_offset
1915  *   l3 packet offset.
1916  * @param l4_offset
1917  *   l4 packet offset.
1918  *
1919  * @return
1920  *   Packet type information.
1921  */
1922 static inline uint64_t
1923 mrvl_desc_to_packet_type_and_offset(struct pp2_ppio_desc *desc,
1924                                     uint8_t *l3_offset, uint8_t *l4_offset)
1925 {
1926         enum pp2_inq_l3_type l3_type;
1927         enum pp2_inq_l4_type l4_type;
1928         uint64_t packet_type;
1929
1930         pp2_ppio_inq_desc_get_l3_info(desc, &l3_type, l3_offset);
1931         pp2_ppio_inq_desc_get_l4_info(desc, &l4_type, l4_offset);
1932
1933         packet_type = RTE_PTYPE_L2_ETHER;
1934
1935         switch (l3_type) {
1936         case PP2_INQ_L3_TYPE_IPV4_NO_OPTS:
1937                 packet_type |= RTE_PTYPE_L3_IPV4;
1938                 break;
1939         case PP2_INQ_L3_TYPE_IPV4_OK:
1940                 packet_type |= RTE_PTYPE_L3_IPV4_EXT;
1941                 break;
1942         case PP2_INQ_L3_TYPE_IPV4_TTL_ZERO:
1943                 packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
1944                 break;
1945         case PP2_INQ_L3_TYPE_IPV6_NO_EXT:
1946                 packet_type |= RTE_PTYPE_L3_IPV6;
1947                 break;
1948         case PP2_INQ_L3_TYPE_IPV6_EXT:
1949                 packet_type |= RTE_PTYPE_L3_IPV6_EXT;
1950                 break;
1951         case PP2_INQ_L3_TYPE_ARP:
1952                 packet_type |= RTE_PTYPE_L2_ETHER_ARP;
1953                 /*
1954                  * In case of ARP l4_offset is set to wrong value.
1955                  * Set it to proper one so that later on mbuf->l3_len can be
1956                  * calculated subtracting l4_offset and l3_offset.
1957                  */
1958                 *l4_offset = *l3_offset + MRVL_ARP_LENGTH;
1959                 break;
1960         default:
1961                 MRVL_LOG(DEBUG, "Failed to recognise l3 packet type");
1962                 break;
1963         }
1964
1965         switch (l4_type) {
1966         case PP2_INQ_L4_TYPE_TCP:
1967                 packet_type |= RTE_PTYPE_L4_TCP;
1968                 break;
1969         case PP2_INQ_L4_TYPE_UDP:
1970                 packet_type |= RTE_PTYPE_L4_UDP;
1971                 break;
1972         default:
1973                 MRVL_LOG(DEBUG, "Failed to recognise l4 packet type");
1974                 break;
1975         }
1976
1977         return packet_type;
1978 }
1979
1980 /**
1981  * Get offload information from the received packet descriptor.
1982  *
1983  * @param desc
1984  *   Pointer to the received packet descriptor.
1985  *
1986  * @return
1987  *   Mbuf offload flags.
1988  */
1989 static inline uint64_t
1990 mrvl_desc_to_ol_flags(struct pp2_ppio_desc *desc)
1991 {
1992         uint64_t flags;
1993         enum pp2_inq_desc_status status;
1994
1995         status = pp2_ppio_inq_desc_get_l3_pkt_error(desc);
1996         if (unlikely(status != PP2_DESC_ERR_OK))
1997                 flags = PKT_RX_IP_CKSUM_BAD;
1998         else
1999                 flags = PKT_RX_IP_CKSUM_GOOD;
2000
2001         status = pp2_ppio_inq_desc_get_l4_pkt_error(desc);
2002         if (unlikely(status != PP2_DESC_ERR_OK))
2003                 flags |= PKT_RX_L4_CKSUM_BAD;
2004         else
2005                 flags |= PKT_RX_L4_CKSUM_GOOD;
2006
2007         return flags;
2008 }
2009
2010 /**
2011  * DPDK callback for receive.
2012  *
2013  * @param rxq
2014  *   Generic pointer to the receive queue.
2015  * @param rx_pkts
2016  *   Array to store received packets.
2017  * @param nb_pkts
2018  *   Maximum number of packets in array.
2019  *
2020  * @return
2021  *   Number of packets successfully received.
2022  */
2023 static uint16_t
2024 mrvl_rx_pkt_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
2025 {
2026         struct mrvl_rxq *q = rxq;
2027         struct pp2_ppio_desc descs[nb_pkts];
2028         struct pp2_bpool *bpool;
2029         int i, ret, rx_done = 0;
2030         int num;
2031         struct pp2_hif *hif;
2032         unsigned int core_id = rte_lcore_id();
2033
2034         hif = mrvl_get_hif(q->priv, core_id);
2035
2036         if (unlikely(!q->priv->ppio || !hif))
2037                 return 0;
2038
2039         bpool = q->priv->bpool;
2040
2041         ret = pp2_ppio_recv(q->priv->ppio, q->priv->rxq_map[q->queue_id].tc,
2042                             q->priv->rxq_map[q->queue_id].inq, descs, &nb_pkts);
2043         if (unlikely(ret < 0)) {
2044                 MRVL_LOG(ERR, "Failed to receive packets");
2045                 return 0;
2046         }
2047         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] -= nb_pkts;
2048
2049         for (i = 0; i < nb_pkts; i++) {
2050                 struct rte_mbuf *mbuf;
2051                 uint8_t l3_offset, l4_offset;
2052                 enum pp2_inq_desc_status status;
2053                 uint64_t addr;
2054
2055                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
2056                         struct pp2_ppio_desc *pref_desc;
2057                         u64 pref_addr;
2058
2059                         pref_desc = &descs[i + MRVL_MUSDK_PREFETCH_SHIFT];
2060                         pref_addr = cookie_addr_high |
2061                                     pp2_ppio_inq_desc_get_cookie(pref_desc);
2062                         rte_mbuf_prefetch_part1((struct rte_mbuf *)(pref_addr));
2063                         rte_mbuf_prefetch_part2((struct rte_mbuf *)(pref_addr));
2064                 }
2065
2066                 addr = cookie_addr_high |
2067                        pp2_ppio_inq_desc_get_cookie(&descs[i]);
2068                 mbuf = (struct rte_mbuf *)addr;
2069                 rte_pktmbuf_reset(mbuf);
2070
2071                 /* drop packet in case of mac, overrun or resource error */
2072                 status = pp2_ppio_inq_desc_get_l2_pkt_error(&descs[i]);
2073                 if (unlikely(status != PP2_DESC_ERR_OK)) {
2074                         struct pp2_buff_inf binf = {
2075                                 .addr = rte_mbuf_data_iova_default(mbuf),
2076                                 .cookie = (pp2_cookie_t)(uint64_t)mbuf,
2077                         };
2078
2079                         pp2_bpool_put_buff(hif, bpool, &binf);
2080                         mrvl_port_bpool_size
2081                                 [bpool->pp2_id][bpool->id][core_id]++;
2082                         q->drop_mac++;
2083                         continue;
2084                 }
2085
2086                 mbuf->data_off += MRVL_PKT_EFFEC_OFFS;
2087                 mbuf->pkt_len = pp2_ppio_inq_desc_get_pkt_len(&descs[i]);
2088                 mbuf->data_len = mbuf->pkt_len;
2089                 mbuf->port = q->port_id;
2090                 mbuf->packet_type =
2091                         mrvl_desc_to_packet_type_and_offset(&descs[i],
2092                                                             &l3_offset,
2093                                                             &l4_offset);
2094                 mbuf->l2_len = l3_offset;
2095                 mbuf->l3_len = l4_offset - l3_offset;
2096
2097                 if (likely(q->cksum_enabled))
2098                         mbuf->ol_flags = mrvl_desc_to_ol_flags(&descs[i]);
2099
2100                 rx_pkts[rx_done++] = mbuf;
2101                 q->bytes_recv += mbuf->pkt_len;
2102         }
2103
2104         if (rte_spinlock_trylock(&q->priv->lock) == 1) {
2105                 num = mrvl_get_bpool_size(bpool->pp2_id, bpool->id);
2106
2107                 if (unlikely(num <= q->priv->bpool_min_size ||
2108                              (!rx_done && num < q->priv->bpool_init_size))) {
2109                         ret = mrvl_fill_bpool(q, MRVL_BURST_SIZE);
2110                         if (ret)
2111                                 MRVL_LOG(ERR, "Failed to fill bpool");
2112                 } else if (unlikely(num > q->priv->bpool_max_size)) {
2113                         int i;
2114                         int pkt_to_remove = num - q->priv->bpool_init_size;
2115                         struct rte_mbuf *mbuf;
2116                         struct pp2_buff_inf buff;
2117
2118                         MRVL_LOG(DEBUG,
2119                                 "port-%d:%d: bpool %d oversize - remove %d buffers (pool size: %d -> %d)",
2120                                 bpool->pp2_id, q->priv->ppio->port_id,
2121                                 bpool->id, pkt_to_remove, num,
2122                                 q->priv->bpool_init_size);
2123
2124                         for (i = 0; i < pkt_to_remove; i++) {
2125                                 ret = pp2_bpool_get_buff(hif, bpool, &buff);
2126                                 if (ret)
2127                                         break;
2128                                 mbuf = (struct rte_mbuf *)
2129                                         (cookie_addr_high | buff.cookie);
2130                                 rte_pktmbuf_free(mbuf);
2131                         }
2132                         mrvl_port_bpool_size
2133                                 [bpool->pp2_id][bpool->id][core_id] -= i;
2134                 }
2135                 rte_spinlock_unlock(&q->priv->lock);
2136         }
2137
2138         return rx_done;
2139 }
2140
2141 /**
2142  * Prepare offload information.
2143  *
2144  * @param ol_flags
2145  *   Offload flags.
2146  * @param packet_type
2147  *   Packet type bitfield.
2148  * @param l3_type
2149  *   Pointer to the pp2_ouq_l3_type structure.
2150  * @param l4_type
2151  *   Pointer to the pp2_outq_l4_type structure.
2152  * @param gen_l3_cksum
2153  *   Will be set to 1 in case l3 checksum is computed.
2154  * @param l4_cksum
2155  *   Will be set to 1 in case l4 checksum is computed.
2156  *
2157  * @return
2158  *   0 on success, negative error value otherwise.
2159  */
2160 static inline int
2161 mrvl_prepare_proto_info(uint64_t ol_flags, uint32_t packet_type,
2162                         enum pp2_outq_l3_type *l3_type,
2163                         enum pp2_outq_l4_type *l4_type,
2164                         int *gen_l3_cksum,
2165                         int *gen_l4_cksum)
2166 {
2167         /*
2168          * Based on ol_flags prepare information
2169          * for pp2_ppio_outq_desc_set_proto_info() which setups descriptor
2170          * for offloading.
2171          */
2172         if (ol_flags & PKT_TX_IPV4) {
2173                 *l3_type = PP2_OUTQ_L3_TYPE_IPV4;
2174                 *gen_l3_cksum = ol_flags & PKT_TX_IP_CKSUM ? 1 : 0;
2175         } else if (ol_flags & PKT_TX_IPV6) {
2176                 *l3_type = PP2_OUTQ_L3_TYPE_IPV6;
2177                 /* no checksum for ipv6 header */
2178                 *gen_l3_cksum = 0;
2179         } else {
2180                 /* if something different then stop processing */
2181                 return -1;
2182         }
2183
2184         ol_flags &= PKT_TX_L4_MASK;
2185         if ((packet_type & RTE_PTYPE_L4_TCP) &&
2186             ol_flags == PKT_TX_TCP_CKSUM) {
2187                 *l4_type = PP2_OUTQ_L4_TYPE_TCP;
2188                 *gen_l4_cksum = 1;
2189         } else if ((packet_type & RTE_PTYPE_L4_UDP) &&
2190                    ol_flags == PKT_TX_UDP_CKSUM) {
2191                 *l4_type = PP2_OUTQ_L4_TYPE_UDP;
2192                 *gen_l4_cksum = 1;
2193         } else {
2194                 *l4_type = PP2_OUTQ_L4_TYPE_OTHER;
2195                 /* no checksum for other type */
2196                 *gen_l4_cksum = 0;
2197         }
2198
2199         return 0;
2200 }
2201
2202 /**
2203  * Release already sent buffers to bpool (buffer-pool).
2204  *
2205  * @param ppio
2206  *   Pointer to the port structure.
2207  * @param hif
2208  *   Pointer to the MUSDK hardware interface.
2209  * @param sq
2210  *   Pointer to the shadow queue.
2211  * @param qid
2212  *   Queue id number.
2213  * @param force
2214  *   Force releasing packets.
2215  */
2216 static inline void
2217 mrvl_free_sent_buffers(struct pp2_ppio *ppio, struct pp2_hif *hif,
2218                        unsigned int core_id, struct mrvl_shadow_txq *sq,
2219                        int qid, int force)
2220 {
2221         struct buff_release_entry *entry;
2222         uint16_t nb_done = 0, num = 0, skip_bufs = 0;
2223         int i;
2224
2225         pp2_ppio_get_num_outq_done(ppio, hif, qid, &nb_done);
2226
2227         sq->num_to_release += nb_done;
2228
2229         if (likely(!force &&
2230                    sq->num_to_release < MRVL_PP2_BUF_RELEASE_BURST_SIZE))
2231                 return;
2232
2233         nb_done = sq->num_to_release;
2234         sq->num_to_release = 0;
2235
2236         for (i = 0; i < nb_done; i++) {
2237                 entry = &sq->ent[sq->tail + num];
2238                 if (unlikely(!entry->buff.addr)) {
2239                         MRVL_LOG(ERR,
2240                                 "Shadow memory @%d: cookie(%lx), pa(%lx)!",
2241                                 sq->tail, (u64)entry->buff.cookie,
2242                                 (u64)entry->buff.addr);
2243                         skip_bufs = 1;
2244                         goto skip;
2245                 }
2246
2247                 if (unlikely(!entry->bpool)) {
2248                         struct rte_mbuf *mbuf;
2249
2250                         mbuf = (struct rte_mbuf *)
2251                                (cookie_addr_high | entry->buff.cookie);
2252                         rte_pktmbuf_free(mbuf);
2253                         skip_bufs = 1;
2254                         goto skip;
2255                 }
2256
2257                 mrvl_port_bpool_size
2258                         [entry->bpool->pp2_id][entry->bpool->id][core_id]++;
2259                 num++;
2260                 if (unlikely(sq->tail + num == MRVL_PP2_TX_SHADOWQ_SIZE))
2261                         goto skip;
2262                 continue;
2263 skip:
2264                 if (likely(num))
2265                         pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
2266                 num += skip_bufs;
2267                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
2268                 sq->size -= num;
2269                 num = 0;
2270                 skip_bufs = 0;
2271         }
2272
2273         if (likely(num)) {
2274                 pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
2275                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
2276                 sq->size -= num;
2277         }
2278 }
2279
2280 /**
2281  * DPDK callback for transmit.
2282  *
2283  * @param txq
2284  *   Generic pointer transmit queue.
2285  * @param tx_pkts
2286  *   Packets to transmit.
2287  * @param nb_pkts
2288  *   Number of packets in array.
2289  *
2290  * @return
2291  *   Number of packets successfully transmitted.
2292  */
2293 static uint16_t
2294 mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
2295 {
2296         struct mrvl_txq *q = txq;
2297         struct mrvl_shadow_txq *sq;
2298         struct pp2_hif *hif;
2299         struct pp2_ppio_desc descs[nb_pkts];
2300         unsigned int core_id = rte_lcore_id();
2301         int i, ret, bytes_sent = 0;
2302         uint16_t num, sq_free_size;
2303         uint64_t addr;
2304
2305         hif = mrvl_get_hif(q->priv, core_id);
2306         sq = &q->shadow_txqs[core_id];
2307
2308         if (unlikely(!q->priv->ppio || !hif))
2309                 return 0;
2310
2311         if (sq->size)
2312                 mrvl_free_sent_buffers(q->priv->ppio, hif, core_id,
2313                                        sq, q->queue_id, 0);
2314
2315         sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1;
2316         if (unlikely(nb_pkts > sq_free_size)) {
2317                 MRVL_LOG(DEBUG,
2318                         "No room in shadow queue for %d packets! %d packets will be sent.",
2319                         nb_pkts, sq_free_size);
2320                 nb_pkts = sq_free_size;
2321         }
2322
2323         for (i = 0; i < nb_pkts; i++) {
2324                 struct rte_mbuf *mbuf = tx_pkts[i];
2325                 int gen_l3_cksum, gen_l4_cksum;
2326                 enum pp2_outq_l3_type l3_type;
2327                 enum pp2_outq_l4_type l4_type;
2328
2329                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
2330                         struct rte_mbuf *pref_pkt_hdr;
2331
2332                         pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT];
2333                         rte_mbuf_prefetch_part1(pref_pkt_hdr);
2334                         rte_mbuf_prefetch_part2(pref_pkt_hdr);
2335                 }
2336
2337                 sq->ent[sq->head].buff.cookie = (pp2_cookie_t)(uint64_t)mbuf;
2338                 sq->ent[sq->head].buff.addr =
2339                         rte_mbuf_data_iova_default(mbuf);
2340                 sq->ent[sq->head].bpool =
2341                         (unlikely(mbuf->port >= RTE_MAX_ETHPORTS ||
2342                          mbuf->refcnt > 1)) ? NULL :
2343                          mrvl_port_to_bpool_lookup[mbuf->port];
2344                 sq->head = (sq->head + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
2345                 sq->size++;
2346
2347                 pp2_ppio_outq_desc_reset(&descs[i]);
2348                 pp2_ppio_outq_desc_set_phys_addr(&descs[i],
2349                                                  rte_pktmbuf_iova(mbuf));
2350                 pp2_ppio_outq_desc_set_pkt_offset(&descs[i], 0);
2351                 pp2_ppio_outq_desc_set_pkt_len(&descs[i],
2352                                                rte_pktmbuf_pkt_len(mbuf));
2353
2354                 bytes_sent += rte_pktmbuf_pkt_len(mbuf);
2355                 /*
2356                  * in case unsupported ol_flags were passed
2357                  * do not update descriptor offload information
2358                  */
2359                 ret = mrvl_prepare_proto_info(mbuf->ol_flags, mbuf->packet_type,
2360                                               &l3_type, &l4_type, &gen_l3_cksum,
2361                                               &gen_l4_cksum);
2362                 if (unlikely(ret))
2363                         continue;
2364
2365                 pp2_ppio_outq_desc_set_proto_info(&descs[i], l3_type, l4_type,
2366                                                   mbuf->l2_len,
2367                                                   mbuf->l2_len + mbuf->l3_len,
2368                                                   gen_l3_cksum, gen_l4_cksum);
2369         }
2370
2371         num = nb_pkts;
2372         pp2_ppio_send(q->priv->ppio, hif, q->queue_id, descs, &nb_pkts);
2373         /* number of packets that were not sent */
2374         if (unlikely(num > nb_pkts)) {
2375                 for (i = nb_pkts; i < num; i++) {
2376                         sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) &
2377                                 MRVL_PP2_TX_SHADOWQ_MASK;
2378                         addr = cookie_addr_high | sq->ent[sq->head].buff.cookie;
2379                         bytes_sent -=
2380                                 rte_pktmbuf_pkt_len((struct rte_mbuf *)addr);
2381                 }
2382                 sq->size -= num - nb_pkts;
2383         }
2384
2385         q->bytes_sent += bytes_sent;
2386
2387         return nb_pkts;
2388 }
2389
2390 /**
2391  * Initialize packet processor.
2392  *
2393  * @return
2394  *   0 on success, negative error value otherwise.
2395  */
2396 static int
2397 mrvl_init_pp2(void)
2398 {
2399         struct pp2_init_params init_params;
2400
2401         memset(&init_params, 0, sizeof(init_params));
2402         init_params.hif_reserved_map = MRVL_MUSDK_HIFS_RESERVED;
2403         init_params.bm_pool_reserved_map = MRVL_MUSDK_BPOOLS_RESERVED;
2404         init_params.rss_tbl_reserved_map = MRVL_MUSDK_RSS_RESERVED;
2405
2406         return pp2_init(&init_params);
2407 }
2408
2409 /**
2410  * Deinitialize packet processor.
2411  *
2412  * @return
2413  *   0 on success, negative error value otherwise.
2414  */
2415 static void
2416 mrvl_deinit_pp2(void)
2417 {
2418         pp2_deinit();
2419 }
2420
2421 /**
2422  * Create private device structure.
2423  *
2424  * @param dev_name
2425  *   Pointer to the port name passed in the initialization parameters.
2426  *
2427  * @return
2428  *   Pointer to the newly allocated private device structure.
2429  */
2430 static struct mrvl_priv *
2431 mrvl_priv_create(const char *dev_name)
2432 {
2433         struct pp2_bpool_params bpool_params;
2434         char match[MRVL_MATCH_LEN];
2435         struct mrvl_priv *priv;
2436         int ret, bpool_bit;
2437
2438         priv = rte_zmalloc_socket(dev_name, sizeof(*priv), 0, rte_socket_id());
2439         if (!priv)
2440                 return NULL;
2441
2442         ret = pp2_netdev_get_ppio_info((char *)(uintptr_t)dev_name,
2443                                        &priv->pp_id, &priv->ppio_id);
2444         if (ret)
2445                 goto out_free_priv;
2446
2447         bpool_bit = mrvl_reserve_bit(&used_bpools[priv->pp_id],
2448                                      PP2_BPOOL_NUM_POOLS);
2449         if (bpool_bit < 0)
2450                 goto out_free_priv;
2451         priv->bpool_bit = bpool_bit;
2452
2453         snprintf(match, sizeof(match), "pool-%d:%d", priv->pp_id,
2454                  priv->bpool_bit);
2455         memset(&bpool_params, 0, sizeof(bpool_params));
2456         bpool_params.match = match;
2457         bpool_params.buff_len = MRVL_PKT_SIZE_MAX + MRVL_PKT_EFFEC_OFFS;
2458         ret = pp2_bpool_init(&bpool_params, &priv->bpool);
2459         if (ret)
2460                 goto out_clear_bpool_bit;
2461
2462         priv->ppio_params.type = PP2_PPIO_T_NIC;
2463         rte_spinlock_init(&priv->lock);
2464
2465         return priv;
2466 out_clear_bpool_bit:
2467         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
2468 out_free_priv:
2469         rte_free(priv);
2470         return NULL;
2471 }
2472
2473 /**
2474  * Create device representing Ethernet port.
2475  *
2476  * @param name
2477  *   Pointer to the port's name.
2478  *
2479  * @return
2480  *   0 on success, negative error value otherwise.
2481  */
2482 static int
2483 mrvl_eth_dev_create(struct rte_vdev_device *vdev, const char *name)
2484 {
2485         int ret, fd = socket(AF_INET, SOCK_DGRAM, 0);
2486         struct rte_eth_dev *eth_dev;
2487         struct mrvl_priv *priv;
2488         struct ifreq req;
2489
2490         eth_dev = rte_eth_dev_allocate(name);
2491         if (!eth_dev)
2492                 return -ENOMEM;
2493
2494         priv = mrvl_priv_create(name);
2495         if (!priv) {
2496                 ret = -ENOMEM;
2497                 goto out_free_dev;
2498         }
2499
2500         eth_dev->data->mac_addrs =
2501                 rte_zmalloc("mac_addrs",
2502                             ETHER_ADDR_LEN * MRVL_MAC_ADDRS_MAX, 0);
2503         if (!eth_dev->data->mac_addrs) {
2504                 MRVL_LOG(ERR, "Failed to allocate space for eth addrs");
2505                 ret = -ENOMEM;
2506                 goto out_free_priv;
2507         }
2508
2509         memset(&req, 0, sizeof(req));
2510         strcpy(req.ifr_name, name);
2511         ret = ioctl(fd, SIOCGIFHWADDR, &req);
2512         if (ret)
2513                 goto out_free_mac;
2514
2515         memcpy(eth_dev->data->mac_addrs[0].addr_bytes,
2516                req.ifr_addr.sa_data, ETHER_ADDR_LEN);
2517
2518         eth_dev->rx_pkt_burst = mrvl_rx_pkt_burst;
2519         eth_dev->tx_pkt_burst = mrvl_tx_pkt_burst;
2520         eth_dev->data->kdrv = RTE_KDRV_NONE;
2521         eth_dev->data->dev_private = priv;
2522         eth_dev->device = &vdev->device;
2523         eth_dev->dev_ops = &mrvl_ops;
2524
2525         rte_eth_dev_probing_finish(eth_dev);
2526         return 0;
2527 out_free_mac:
2528         rte_free(eth_dev->data->mac_addrs);
2529 out_free_dev:
2530         rte_eth_dev_release_port(eth_dev);
2531 out_free_priv:
2532         rte_free(priv);
2533
2534         return ret;
2535 }
2536
2537 /**
2538  * Cleanup previously created device representing Ethernet port.
2539  *
2540  * @param name
2541  *   Pointer to the port name.
2542  */
2543 static void
2544 mrvl_eth_dev_destroy(const char *name)
2545 {
2546         struct rte_eth_dev *eth_dev;
2547         struct mrvl_priv *priv;
2548
2549         eth_dev = rte_eth_dev_allocated(name);
2550         if (!eth_dev)
2551                 return;
2552
2553         priv = eth_dev->data->dev_private;
2554         pp2_bpool_deinit(priv->bpool);
2555         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
2556         rte_free(priv);
2557         rte_free(eth_dev->data->mac_addrs);
2558         rte_eth_dev_release_port(eth_dev);
2559 }
2560
2561 /**
2562  * Callback used by rte_kvargs_process() during argument parsing.
2563  *
2564  * @param key
2565  *   Pointer to the parsed key (unused).
2566  * @param value
2567  *   Pointer to the parsed value.
2568  * @param extra_args
2569  *   Pointer to the extra arguments which contains address of the
2570  *   table of pointers to parsed interface names.
2571  *
2572  * @return
2573  *   Always 0.
2574  */
2575 static int
2576 mrvl_get_ifnames(const char *key __rte_unused, const char *value,
2577                  void *extra_args)
2578 {
2579         struct mrvl_ifnames *ifnames = extra_args;
2580
2581         ifnames->names[ifnames->idx++] = value;
2582
2583         return 0;
2584 }
2585
2586 /**
2587  * Deinitialize per-lcore MUSDK hardware interfaces (hifs).
2588  */
2589 static void
2590 mrvl_deinit_hifs(void)
2591 {
2592         int i;
2593
2594         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++) {
2595                 if (hifs[i])
2596                         pp2_hif_deinit(hifs[i]);
2597         }
2598         used_hifs = MRVL_MUSDK_HIFS_RESERVED;
2599         memset(hifs, 0, sizeof(hifs));
2600 }
2601
2602 /**
2603  * DPDK callback to register the virtual device.
2604  *
2605  * @param vdev
2606  *   Pointer to the virtual device.
2607  *
2608  * @return
2609  *   0 on success, negative error value otherwise.
2610  */
2611 static int
2612 rte_pmd_mrvl_probe(struct rte_vdev_device *vdev)
2613 {
2614         struct rte_kvargs *kvlist;
2615         struct mrvl_ifnames ifnames;
2616         int ret = -EINVAL;
2617         uint32_t i, ifnum, cfgnum;
2618         const char *params;
2619
2620         params = rte_vdev_device_args(vdev);
2621         if (!params)
2622                 return -EINVAL;
2623
2624         kvlist = rte_kvargs_parse(params, valid_args);
2625         if (!kvlist)
2626                 return -EINVAL;
2627
2628         ifnum = rte_kvargs_count(kvlist, MRVL_IFACE_NAME_ARG);
2629         if (ifnum > RTE_DIM(ifnames.names))
2630                 goto out_free_kvlist;
2631
2632         ifnames.idx = 0;
2633         rte_kvargs_process(kvlist, MRVL_IFACE_NAME_ARG,
2634                            mrvl_get_ifnames, &ifnames);
2635
2636
2637         /*
2638          * The below system initialization should be done only once,
2639          * on the first provided configuration file
2640          */
2641         if (!mrvl_qos_cfg) {
2642                 cfgnum = rte_kvargs_count(kvlist, MRVL_CFG_ARG);
2643                 MRVL_LOG(INFO, "Parsing config file!");
2644                 if (cfgnum > 1) {
2645                         MRVL_LOG(ERR, "Cannot handle more than one config file!");
2646                         goto out_free_kvlist;
2647                 } else if (cfgnum == 1) {
2648                         rte_kvargs_process(kvlist, MRVL_CFG_ARG,
2649                                            mrvl_get_qoscfg, &mrvl_qos_cfg);
2650                 }
2651         }
2652
2653         if (mrvl_dev_num)
2654                 goto init_devices;
2655
2656         MRVL_LOG(INFO, "Perform MUSDK initializations");
2657         /*
2658          * ret == -EEXIST is correct, it means DMA
2659          * has been already initialized (by another PMD).
2660          */
2661         ret = mv_sys_dma_mem_init(MRVL_MUSDK_DMA_MEMSIZE);
2662         if (ret < 0) {
2663                 if (ret != -EEXIST)
2664                         goto out_free_kvlist;
2665                 else
2666                         MRVL_LOG(INFO,
2667                                 "DMA memory has been already initialized by a different driver.");
2668         }
2669
2670         ret = mrvl_init_pp2();
2671         if (ret) {
2672                 MRVL_LOG(ERR, "Failed to init PP!");
2673                 goto out_deinit_dma;
2674         }
2675
2676         memset(mrvl_port_bpool_size, 0, sizeof(mrvl_port_bpool_size));
2677         memset(mrvl_port_to_bpool_lookup, 0, sizeof(mrvl_port_to_bpool_lookup));
2678
2679         mrvl_lcore_first = RTE_MAX_LCORE;
2680         mrvl_lcore_last = 0;
2681
2682 init_devices:
2683         for (i = 0; i < ifnum; i++) {
2684                 MRVL_LOG(INFO, "Creating %s", ifnames.names[i]);
2685                 ret = mrvl_eth_dev_create(vdev, ifnames.names[i]);
2686                 if (ret)
2687                         goto out_cleanup;
2688         }
2689         mrvl_dev_num += ifnum;
2690
2691         rte_kvargs_free(kvlist);
2692
2693         return 0;
2694 out_cleanup:
2695         for (; i > 0; i--)
2696                 mrvl_eth_dev_destroy(ifnames.names[i]);
2697
2698         if (mrvl_dev_num == 0)
2699                 mrvl_deinit_pp2();
2700 out_deinit_dma:
2701         if (mrvl_dev_num == 0)
2702                 mv_sys_dma_mem_destroy();
2703 out_free_kvlist:
2704         rte_kvargs_free(kvlist);
2705
2706         return ret;
2707 }
2708
2709 /**
2710  * DPDK callback to remove virtual device.
2711  *
2712  * @param vdev
2713  *   Pointer to the removed virtual device.
2714  *
2715  * @return
2716  *   0 on success, negative error value otherwise.
2717  */
2718 static int
2719 rte_pmd_mrvl_remove(struct rte_vdev_device *vdev)
2720 {
2721         int i;
2722         const char *name;
2723
2724         name = rte_vdev_device_name(vdev);
2725         if (!name)
2726                 return -EINVAL;
2727
2728         MRVL_LOG(INFO, "Removing %s", name);
2729
2730         RTE_ETH_FOREACH_DEV(i) { /* FIXME: removing all devices! */
2731                 char ifname[RTE_ETH_NAME_MAX_LEN];
2732
2733                 rte_eth_dev_get_name_by_port(i, ifname);
2734                 mrvl_eth_dev_destroy(ifname);
2735                 mrvl_dev_num--;
2736         }
2737
2738         if (mrvl_dev_num == 0) {
2739                 MRVL_LOG(INFO, "Perform MUSDK deinit");
2740                 mrvl_deinit_hifs();
2741                 mrvl_deinit_pp2();
2742                 mv_sys_dma_mem_destroy();
2743         }
2744
2745         return 0;
2746 }
2747
2748 static struct rte_vdev_driver pmd_mrvl_drv = {
2749         .probe = rte_pmd_mrvl_probe,
2750         .remove = rte_pmd_mrvl_remove,
2751 };
2752
2753 RTE_PMD_REGISTER_VDEV(net_mvpp2, pmd_mrvl_drv);
2754 RTE_PMD_REGISTER_ALIAS(net_mvpp2, eth_mvpp2);
2755
2756 RTE_INIT(mrvl_init_log)
2757 {
2758         mrvl_logtype = rte_log_register("pmd.net.mvpp2");
2759         if (mrvl_logtype >= 0)
2760                 rte_log_set_level(mrvl_logtype, RTE_LOG_NOTICE);
2761 }