drivers/net/mvpp2/mrvl_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Marvell International Ltd.
   3  * Copyright(c) 2017 Semihalf.
   4  * All rights reserved.
   5  */
   6
   7 #include <rte_ethdev_driver.h>
   8 #include <rte_kvargs.h>
   9 #include <rte_log.h>
  10 #include <rte_malloc.h>
  11 #include <rte_bus_vdev.h>
  12
  13 #include <fcntl.h>
  14 #include <linux/ethtool.h>
  15 #include <linux/sockios.h>
  16 #include <net/if.h>
  17 #include <net/if_arp.h>
  18 #include <sys/ioctl.h>
  19 #include <sys/socket.h>
  20 #include <sys/stat.h>
  21 #include <sys/types.h>
  22
  23 #include <rte_mvep_common.h>
  24 #include "mrvl_ethdev.h"
  25 #include "mrvl_qos.h"
  26 #include "mrvl_flow.h"
  27 #include "mrvl_mtr.h"
  28 #include "mrvl_tm.h"
  29
  30 /* bitmask with reserved hifs */
  31 #define MRVL_MUSDK_HIFS_RESERVED 0x0F
  32 /* bitmask with reserved bpools */
  33 #define MRVL_MUSDK_BPOOLS_RESERVED 0x07
  34 /* bitmask with reserved kernel RSS tables */
  35 #define MRVL_MUSDK_RSS_RESERVED 0x01
  36 /* maximum number of available hifs */
  37 #define MRVL_MUSDK_HIFS_MAX 9
  38
  39 /* prefetch shift */
  40 #define MRVL_MUSDK_PREFETCH_SHIFT 2
  41
  42 /* TCAM has 25 entries reserved for uc/mc filter entries */
  43 #define MRVL_MAC_ADDRS_MAX 25
  44 #define MRVL_MATCH_LEN 16
  45 #define MRVL_PKT_EFFEC_OFFS (MRVL_PKT_OFFS + MV_MH_SIZE)
  46 /* Maximum allowable packet size */
  47 #define MRVL_PKT_SIZE_MAX (10240 - MV_MH_SIZE)
  48
  49 #define MRVL_IFACE_NAME_ARG "iface"
  50 #define MRVL_CFG_ARG "cfg"
  51
  52 #define MRVL_BURST_SIZE 64
  53
  54 #define MRVL_ARP_LENGTH 28
  55
  56 #define MRVL_COOKIE_ADDR_INVALID ~0ULL
  57 #define MRVL_COOKIE_HIGH_ADDR_MASK 0xffffff0000000000
  58
  59 /** Port Rx offload capabilities */
  60 #define MRVL_RX_OFFLOADS (DEV_RX_OFFLOAD_VLAN_FILTER | \
  61                           DEV_RX_OFFLOAD_JUMBO_FRAME | \
  62                           DEV_RX_OFFLOAD_CHECKSUM)
  63
  64 /** Port Tx offloads capabilities */
  65 #define MRVL_TX_OFFLOADS (DEV_TX_OFFLOAD_IPV4_CKSUM | \
  66                           DEV_TX_OFFLOAD_UDP_CKSUM | \
  67                           DEV_TX_OFFLOAD_TCP_CKSUM | \
  68                           DEV_TX_OFFLOAD_MULTI_SEGS)
  69
  70 static const char * const valid_args[] = {
  71         MRVL_IFACE_NAME_ARG,
  72         MRVL_CFG_ARG,
  73         NULL
  74 };
  75
  76 static int used_hifs = MRVL_MUSDK_HIFS_RESERVED;
  77 static struct pp2_hif *hifs[RTE_MAX_LCORE];
  78 static int used_bpools[PP2_NUM_PKT_PROC] = {
  79         [0 ... PP2_NUM_PKT_PROC - 1] = MRVL_MUSDK_BPOOLS_RESERVED
  80 };
  81
  82 static struct pp2_bpool *mrvl_port_to_bpool_lookup[RTE_MAX_ETHPORTS];
  83 static int mrvl_port_bpool_size[PP2_NUM_PKT_PROC][PP2_BPOOL_NUM_POOLS][RTE_MAX_LCORE];
  84 static uint64_t cookie_addr_high = MRVL_COOKIE_ADDR_INVALID;
  85
  86 int mrvl_logtype;
  87
  88 struct mrvl_ifnames {
  89         const char *names[PP2_NUM_ETH_PPIO * PP2_NUM_PKT_PROC];
  90         int idx;
  91 };
  92
  93 /*
  94  * To use buffer harvesting based on loopback port shadow queue structure
  95  * was introduced for buffers information bookkeeping.
  96  *
  97  * Before sending the packet, related buffer information (pp2_buff_inf) is
  98  * stored in shadow queue. After packet is transmitted no longer used
  99  * packet buffer is released back to it's original hardware pool,
 100  * on condition it originated from interface.
 101  * In case it  was generated by application itself i.e: mbuf->port field is
 102  * 0xff then its released to software mempool.
 103  */
 104 struct mrvl_shadow_txq {
 105         int head;           /* write index - used when sending buffers */
 106         int tail;           /* read index - used when releasing buffers */
 107         u16 size;           /* queue occupied size */
 108         u16 num_to_release; /* number of descriptors sent, that can be
 109                              * released
 110                              */
 111         struct buff_release_entry ent[MRVL_PP2_TX_SHADOWQ_SIZE]; /* q entries */
 112 };
 113
 114 struct mrvl_rxq {
 115         struct mrvl_priv *priv;
 116         struct rte_mempool *mp;
 117         int queue_id;
 118         int port_id;
 119         int cksum_enabled;
 120         uint64_t bytes_recv;
 121         uint64_t drop_mac;
 122 };
 123
 124 struct mrvl_txq {
 125         struct mrvl_priv *priv;
 126         int queue_id;
 127         int port_id;
 128         uint64_t bytes_sent;
 129         struct mrvl_shadow_txq shadow_txqs[RTE_MAX_LCORE];
 130         int tx_deferred_start;
 131 };
 132
 133 static int mrvl_lcore_first;
 134 static int mrvl_lcore_last;
 135 static int mrvl_dev_num;
 136
 137 static int mrvl_fill_bpool(struct mrvl_rxq *rxq, int num);
 138 static inline void mrvl_free_sent_buffers(struct pp2_ppio *ppio,
 139                         struct pp2_hif *hif, unsigned int core_id,
 140                         struct mrvl_shadow_txq *sq, int qid, int force);
 141
 142 static uint16_t mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts,
 143                                   uint16_t nb_pkts);
 144 static uint16_t mrvl_tx_sg_pkt_burst(void *txq, struct rte_mbuf **tx_pkts,
 145                                      uint16_t nb_pkts);
 146
 147
 148 #define MRVL_XSTATS_TBL_ENTRY(name) { \
 149         #name, offsetof(struct pp2_ppio_statistics, name),      \
 150         sizeof(((struct pp2_ppio_statistics *)0)->name)         \
 151 }
 152
 153 /* Table with xstats data */
 154 static struct {
 155         const char *name;
 156         unsigned int offset;
 157         unsigned int size;
 158 } mrvl_xstats_tbl[] = {
 159         MRVL_XSTATS_TBL_ENTRY(rx_bytes),
 160         MRVL_XSTATS_TBL_ENTRY(rx_packets),
 161         MRVL_XSTATS_TBL_ENTRY(rx_unicast_packets),
 162         MRVL_XSTATS_TBL_ENTRY(rx_errors),
 163         MRVL_XSTATS_TBL_ENTRY(rx_fullq_dropped),
 164         MRVL_XSTATS_TBL_ENTRY(rx_bm_dropped),
 165         MRVL_XSTATS_TBL_ENTRY(rx_early_dropped),
 166         MRVL_XSTATS_TBL_ENTRY(rx_fifo_dropped),
 167         MRVL_XSTATS_TBL_ENTRY(rx_cls_dropped),
 168         MRVL_XSTATS_TBL_ENTRY(tx_bytes),
 169         MRVL_XSTATS_TBL_ENTRY(tx_packets),
 170         MRVL_XSTATS_TBL_ENTRY(tx_unicast_packets),
 171         MRVL_XSTATS_TBL_ENTRY(tx_errors)
 172 };
 173
 174 static inline void
 175 mrvl_fill_shadowq(struct mrvl_shadow_txq *sq, struct rte_mbuf *buf)
 176 {
 177         sq->ent[sq->head].buff.cookie = (uint64_t)buf;
 178         sq->ent[sq->head].buff.addr = buf ?
 179                 rte_mbuf_data_iova_default(buf) : 0;
 180
 181         sq->ent[sq->head].bpool =
 182                 (unlikely(!buf || buf->port >= RTE_MAX_ETHPORTS ||
 183                  buf->refcnt > 1)) ? NULL :
 184                  mrvl_port_to_bpool_lookup[buf->port];
 185
 186         sq->head = (sq->head + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
 187         sq->size++;
 188 }
 189
 190 static inline void
 191 mrvl_fill_desc(struct pp2_ppio_desc *desc, struct rte_mbuf *buf)
 192 {
 193         pp2_ppio_outq_desc_reset(desc);
 194         pp2_ppio_outq_desc_set_phys_addr(desc, rte_pktmbuf_iova(buf));
 195         pp2_ppio_outq_desc_set_pkt_offset(desc, 0);
 196         pp2_ppio_outq_desc_set_pkt_len(desc, rte_pktmbuf_data_len(buf));
 197 }
 198
 199 static inline int
 200 mrvl_get_bpool_size(int pp2_id, int pool_id)
 201 {
 202         int i;
 203         int size = 0;
 204
 205         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++)
 206                 size += mrvl_port_bpool_size[pp2_id][pool_id][i];
 207
 208         return size;
 209 }
 210
 211 static inline int
 212 mrvl_reserve_bit(int *bitmap, int max)
 213 {
 214         int n = sizeof(*bitmap) * 8 - __builtin_clz(*bitmap);
 215
 216         if (n >= max)
 217                 return -1;
 218
 219         *bitmap |= 1 << n;
 220
 221         return n;
 222 }
 223
 224 static int
 225 mrvl_init_hif(int core_id)
 226 {
 227         struct pp2_hif_params params;
 228         char match[MRVL_MATCH_LEN];
 229         int ret;
 230
 231         ret = mrvl_reserve_bit(&used_hifs, MRVL_MUSDK_HIFS_MAX);
 232         if (ret < 0) {
 233                 MRVL_LOG(ERR, "Failed to allocate hif %d", core_id);
 234                 return ret;
 235         }
 236
 237         snprintf(match, sizeof(match), "hif-%d", ret);
 238         memset(&params, 0, sizeof(params));
 239         params.match = match;
 240         params.out_size = MRVL_PP2_AGGR_TXQD_MAX;
 241         ret = pp2_hif_init(&params, &hifs[core_id]);
 242         if (ret) {
 243                 MRVL_LOG(ERR, "Failed to initialize hif %d", core_id);
 244                 return ret;
 245         }
 246
 247         return 0;
 248 }
 249
 250 static inline struct pp2_hif*
 251 mrvl_get_hif(struct mrvl_priv *priv, int core_id)
 252 {
 253         int ret;
 254
 255         if (likely(hifs[core_id] != NULL))
 256                 return hifs[core_id];
 257
 258         rte_spinlock_lock(&priv->lock);
 259
 260         ret = mrvl_init_hif(core_id);
 261         if (ret < 0) {
 262                 MRVL_LOG(ERR, "Failed to allocate hif %d", core_id);
 263                 goto out;
 264         }
 265
 266         if (core_id < mrvl_lcore_first)
 267                 mrvl_lcore_first = core_id;
 268
 269         if (core_id > mrvl_lcore_last)
 270                 mrvl_lcore_last = core_id;
 271 out:
 272         rte_spinlock_unlock(&priv->lock);
 273
 274         return hifs[core_id];
 275 }
 276
 277 /**
 278  * Set tx burst function according to offload flag
 279  *
 280  * @param dev
 281  *   Pointer to Ethernet device structure.
 282  */
 283 static void
 284 mrvl_set_tx_function(struct rte_eth_dev *dev)
 285 {
 286         struct mrvl_priv *priv = dev->data->dev_private;
 287
 288         /* Use a simple Tx queue (no offloads, no multi segs) if possible */
 289         if (priv->multiseg) {
 290                 RTE_LOG(INFO, PMD, "Using multi-segment tx callback\n");
 291                 dev->tx_pkt_burst = mrvl_tx_sg_pkt_burst;
 292         } else {
 293                 RTE_LOG(INFO, PMD, "Using single-segment tx callback\n");
 294                 dev->tx_pkt_burst = mrvl_tx_pkt_burst;
 295         }
 296 }
 297
 298 /**
 299  * Configure rss based on dpdk rss configuration.
 300  *
 301  * @param priv
 302  *   Pointer to private structure.
 303  * @param rss_conf
 304  *   Pointer to RSS configuration.
 305  *
 306  * @return
 307  *   0 on success, negative error value otherwise.
 308  */
 309 static int
 310 mrvl_configure_rss(struct mrvl_priv *priv, struct rte_eth_rss_conf *rss_conf)
 311 {
 312         if (rss_conf->rss_key)
 313                 MRVL_LOG(WARNING, "Changing hash key is not supported");
 314
 315         if (rss_conf->rss_hf == 0) {
 316                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 317         } else if (rss_conf->rss_hf & ETH_RSS_IPV4) {
 318                 priv->ppio_params.inqs_params.hash_type =
 319                         PP2_PPIO_HASH_T_2_TUPLE;
 320         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) {
 321                 priv->ppio_params.inqs_params.hash_type =
 322                         PP2_PPIO_HASH_T_5_TUPLE;
 323                 priv->rss_hf_tcp = 1;
 324         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
 325                 priv->ppio_params.inqs_params.hash_type =
 326                         PP2_PPIO_HASH_T_5_TUPLE;
 327                 priv->rss_hf_tcp = 0;
 328         } else {
 329                 return -EINVAL;
 330         }
 331
 332         return 0;
 333 }
 334
 335 /**
 336  * Ethernet device configuration.
 337  *
 338  * Prepare the driver for a given number of TX and RX queues and
 339  * configure RSS.
 340  *
 341  * @param dev
 342  *   Pointer to Ethernet device structure.
 343  *
 344  * @return
 345  *   0 on success, negative error value otherwise.
 346  */
 347 static int
 348 mrvl_dev_configure(struct rte_eth_dev *dev)
 349 {
 350         struct mrvl_priv *priv = dev->data->dev_private;
 351         int ret;
 352
 353         if (priv->ppio) {
 354                 MRVL_LOG(INFO, "Device reconfiguration is not supported");
 355                 return -EINVAL;
 356         }
 357
 358         if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_NONE &&
 359             dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) {
 360                 MRVL_LOG(INFO, "Unsupported rx multi queue mode %d",
 361                         dev->data->dev_conf.rxmode.mq_mode);
 362                 return -EINVAL;
 363         }
 364
 365         if (dev->data->dev_conf.rxmode.split_hdr_size) {
 366                 MRVL_LOG(INFO, "Split headers not supported");
 367                 return -EINVAL;
 368         }
 369
 370         if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME)
 371                 dev->data->mtu = dev->data->dev_conf.rxmode.max_rx_pkt_len -
 372                                  MRVL_PP2_ETH_HDRS_LEN;
 373
 374         if (dev->data->dev_conf.txmode.offloads & DEV_TX_OFFLOAD_MULTI_SEGS)
 375                 priv->multiseg = 1;
 376
 377         ret = mrvl_configure_rxqs(priv, dev->data->port_id,
 378                                   dev->data->nb_rx_queues);
 379         if (ret < 0)
 380                 return ret;
 381
 382         ret = mrvl_configure_txqs(priv, dev->data->port_id,
 383                                   dev->data->nb_tx_queues);
 384         if (ret < 0)
 385                 return ret;
 386
 387         priv->ppio_params.outqs_params.num_outqs = dev->data->nb_tx_queues;
 388         priv->ppio_params.maintain_stats = 1;
 389         priv->nb_rx_queues = dev->data->nb_rx_queues;
 390
 391         ret = mrvl_tm_init(dev);
 392         if (ret < 0)
 393                 return ret;
 394
 395         if (dev->data->nb_rx_queues == 1 &&
 396             dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) {
 397                 MRVL_LOG(WARNING, "Disabling hash for 1 rx queue");
 398                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 399
 400                 return 0;
 401         }
 402
 403         return mrvl_configure_rss(priv,
 404                                   &dev->data->dev_conf.rx_adv_conf.rss_conf);
 405 }
 406
 407 /**
 408  * DPDK callback to change the MTU.
 409  *
 410  * Setting the MTU affects hardware MRU (packets larger than the MRU
 411  * will be dropped).
 412  *
 413  * @param dev
 414  *   Pointer to Ethernet device structure.
 415  * @param mtu
 416  *   New MTU.
 417  *
 418  * @return
 419  *   0 on success, negative error value otherwise.
 420  */
 421 static int
 422 mrvl_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 423 {
 424         struct mrvl_priv *priv = dev->data->dev_private;
 425         uint16_t mru;
 426         uint16_t mbuf_data_size = 0; /* SW buffer size */
 427         int ret;
 428
 429         mru = MRVL_PP2_MTU_TO_MRU(mtu);
 430         /*
 431          * min_rx_buf_size is equal to mbuf data size
 432          * if pmd didn't set it differently
 433          */
 434         mbuf_data_size = dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM;
 435         /* Prevent PMD from:
 436          * - setting mru greater than the mbuf size resulting in
 437          * hw and sw buffer size mismatch
 438          * - setting mtu that requires the support of scattered packets
 439          * when this feature has not been enabled/supported so far
 440          * (TODO check scattered_rx flag here once scattered RX is supported).
 441          */
 442         if (mru + MRVL_PKT_OFFS > mbuf_data_size) {
 443                 mru = mbuf_data_size - MRVL_PKT_OFFS;
 444                 mtu = MRVL_PP2_MRU_TO_MTU(mru);
 445                 MRVL_LOG(WARNING, "MTU too big, max MTU possible limitted "
 446                         "by current mbuf size: %u. Set MTU to %u, MRU to %u",
 447                         mbuf_data_size, mtu, mru);
 448         }
 449
 450         if (mtu < ETHER_MIN_MTU || mru > MRVL_PKT_SIZE_MAX) {
 451                 MRVL_LOG(ERR, "Invalid MTU [%u] or MRU [%u]", mtu, mru);
 452                 return -EINVAL;
 453         }
 454
 455         dev->data->mtu = mtu;
 456         dev->data->dev_conf.rxmode.max_rx_pkt_len = mru - MV_MH_SIZE;
 457
 458         if (!priv->ppio)
 459                 return 0;
 460
 461         ret = pp2_ppio_set_mru(priv->ppio, mru);
 462         if (ret) {
 463                 MRVL_LOG(ERR, "Failed to change MRU");
 464                 return ret;
 465         }
 466
 467         ret = pp2_ppio_set_mtu(priv->ppio, mtu);
 468         if (ret) {
 469                 MRVL_LOG(ERR, "Failed to change MTU");
 470                 return ret;
 471         }
 472
 473         return 0;
 474 }
 475
 476 /**
 477  * DPDK callback to bring the link up.
 478  *
 479  * @param dev
 480  *   Pointer to Ethernet device structure.
 481  *
 482  * @return
 483  *   0 on success, negative error value otherwise.
 484  */
 485 static int
 486 mrvl_dev_set_link_up(struct rte_eth_dev *dev)
 487 {
 488         struct mrvl_priv *priv = dev->data->dev_private;
 489         int ret;
 490
 491         if (!priv->ppio)
 492                 return -EPERM;
 493
 494         ret = pp2_ppio_enable(priv->ppio);
 495         if (ret)
 496                 return ret;
 497
 498         /*
 499          * mtu/mru can be updated if pp2_ppio_enable() was called at least once
 500          * as pp2_ppio_enable() changes port->t_mode from default 0 to
 501          * PP2_TRAFFIC_INGRESS_EGRESS.
 502          *
 503          * Set mtu to default DPDK value here.
 504          */
 505         ret = mrvl_mtu_set(dev, dev->data->mtu);
 506         if (ret)
 507                 pp2_ppio_disable(priv->ppio);
 508
 509         return ret;
 510 }
 511
 512 /**
 513  * DPDK callback to bring the link down.
 514  *
 515  * @param dev
 516  *   Pointer to Ethernet device structure.
 517  *
 518  * @return
 519  *   0 on success, negative error value otherwise.
 520  */
 521 static int
 522 mrvl_dev_set_link_down(struct rte_eth_dev *dev)
 523 {
 524         struct mrvl_priv *priv = dev->data->dev_private;
 525
 526         if (!priv->ppio)
 527                 return -EPERM;
 528
 529         return pp2_ppio_disable(priv->ppio);
 530 }
 531
 532 /**
 533  * DPDK callback to start tx queue.
 534  *
 535  * @param dev
 536  *   Pointer to Ethernet device structure.
 537  * @param queue_id
 538  *   Transmit queue index.
 539  *
 540  * @return
 541  *   0 on success, negative error value otherwise.
 542  */
 543 static int
 544 mrvl_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id)
 545 {
 546         struct mrvl_priv *priv = dev->data->dev_private;
 547         int ret;
 548
 549         if (!priv)
 550                 return -EPERM;
 551
 552         /* passing 1 enables given tx queue */
 553         ret = pp2_ppio_set_outq_state(priv->ppio, queue_id, 1);
 554         if (ret) {
 555                 MRVL_LOG(ERR, "Failed to start txq %d", queue_id);
 556                 return ret;
 557         }
 558
 559         dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 560
 561         return 0;
 562 }
 563
 564 /**
 565  * DPDK callback to stop tx queue.
 566  *
 567  * @param dev
 568  *   Pointer to Ethernet device structure.
 569  * @param queue_id
 570  *   Transmit queue index.
 571  *
 572  * @return
 573  *   0 on success, negative error value otherwise.
 574  */
 575 static int
 576 mrvl_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id)
 577 {
 578         struct mrvl_priv *priv = dev->data->dev_private;
 579         int ret;
 580
 581         if (!priv->ppio)
 582                 return -EPERM;
 583
 584         /* passing 0 disables given tx queue */
 585         ret = pp2_ppio_set_outq_state(priv->ppio, queue_id, 0);
 586         if (ret) {
 587                 MRVL_LOG(ERR, "Failed to stop txq %d", queue_id);
 588                 return ret;
 589         }
 590
 591         dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 592
 593         return 0;
 594 }
 595
 596 /**
 597  * DPDK callback to start the device.
 598  *
 599  * @param dev
 600  *   Pointer to Ethernet device structure.
 601  *
 602  * @return
 603  *   0 on success, negative errno value on failure.
 604  */
 605 static int
 606 mrvl_dev_start(struct rte_eth_dev *dev)
 607 {
 608         struct mrvl_priv *priv = dev->data->dev_private;
 609         char match[MRVL_MATCH_LEN];
 610         int ret = 0, i, def_init_size;
 611
 612         if (priv->ppio)
 613                 return mrvl_dev_set_link_up(dev);
 614
 615         snprintf(match, sizeof(match), "ppio-%d:%d",
 616                  priv->pp_id, priv->ppio_id);
 617         priv->ppio_params.match = match;
 618
 619         /*
 620          * Calculate the minimum bpool size for refill feature as follows:
 621          * 2 default burst sizes multiply by number of rx queues.
 622          * If the bpool size will be below this value, new buffers will
 623          * be added to the pool.
 624          */
 625         priv->bpool_min_size = priv->nb_rx_queues * MRVL_BURST_SIZE * 2;
 626
 627         /* In case initial bpool size configured in queues setup is
 628          * smaller than minimum size add more buffers
 629          */
 630         def_init_size = priv->bpool_min_size + MRVL_BURST_SIZE * 2;
 631         if (priv->bpool_init_size < def_init_size) {
 632                 int buffs_to_add = def_init_size - priv->bpool_init_size;
 633
 634                 priv->bpool_init_size += buffs_to_add;
 635                 ret = mrvl_fill_bpool(dev->data->rx_queues[0], buffs_to_add);
 636                 if (ret)
 637                         MRVL_LOG(ERR, "Failed to add buffers to bpool");
 638         }
 639
 640         /*
 641          * Calculate the maximum bpool size for refill feature as follows:
 642          * maximum number of descriptors in rx queue multiply by number
 643          * of rx queues plus minimum bpool size.
 644          * In case the bpool size will exceed this value, superfluous buffers
 645          * will be removed
 646          */
 647         priv->bpool_max_size = (priv->nb_rx_queues * MRVL_PP2_RXD_MAX) +
 648                                 priv->bpool_min_size;
 649
 650         ret = pp2_ppio_init(&priv->ppio_params, &priv->ppio);
 651         if (ret) {
 652                 MRVL_LOG(ERR, "Failed to init ppio");
 653                 return ret;
 654         }
 655
 656         /*
 657          * In case there are some some stale uc/mc mac addresses flush them
 658          * here. It cannot be done during mrvl_dev_close() as port information
 659          * is already gone at that point (due to pp2_ppio_deinit() in
 660          * mrvl_dev_stop()).
 661          */
 662         if (!priv->uc_mc_flushed) {
 663                 ret = pp2_ppio_flush_mac_addrs(priv->ppio, 1, 1);
 664                 if (ret) {
 665                         MRVL_LOG(ERR,
 666                                 "Failed to flush uc/mc filter list");
 667                         goto out;
 668                 }
 669                 priv->uc_mc_flushed = 1;
 670         }
 671
 672         if (!priv->vlan_flushed) {
 673                 ret = pp2_ppio_flush_vlan(priv->ppio);
 674                 if (ret) {
 675                         MRVL_LOG(ERR, "Failed to flush vlan list");
 676                         /*
 677                          * TODO
 678                          * once pp2_ppio_flush_vlan() is supported jump to out
 679                          * goto out;
 680                          */
 681                 }
 682                 priv->vlan_flushed = 1;
 683         }
 684         ret = mrvl_mtu_set(dev, dev->data->mtu);
 685         if (ret)
 686                 MRVL_LOG(ERR, "Failed to set MTU to %d", dev->data->mtu);
 687
 688         /* For default QoS config, don't start classifier. */
 689         if (mrvl_qos_cfg  &&
 690             mrvl_qos_cfg->port[dev->data->port_id].use_global_defaults == 0) {
 691                 ret = mrvl_start_qos_mapping(priv);
 692                 if (ret) {
 693                         MRVL_LOG(ERR, "Failed to setup QoS mapping");
 694                         goto out;
 695                 }
 696         }
 697
 698         ret = mrvl_dev_set_link_up(dev);
 699         if (ret) {
 700                 MRVL_LOG(ERR, "Failed to set link up");
 701                 goto out;
 702         }
 703
 704         /* start tx queues */
 705         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 706                 struct mrvl_txq *txq = dev->data->tx_queues[i];
 707
 708                 dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED;
 709
 710                 if (!txq->tx_deferred_start)
 711                         continue;
 712
 713                 /*
 714                  * All txqs are started by default. Stop them
 715                  * so that tx_deferred_start works as expected.
 716                  */
 717                 ret = mrvl_tx_queue_stop(dev, i);
 718                 if (ret)
 719                         goto out;
 720         }
 721
 722         mrvl_flow_init(dev);
 723         mrvl_mtr_init(dev);
 724         mrvl_set_tx_function(dev);
 725
 726         return 0;
 727 out:
 728         MRVL_LOG(ERR, "Failed to start device");
 729         pp2_ppio_deinit(priv->ppio);
 730         return ret;
 731 }
 732
 733 /**
 734  * Flush receive queues.
 735  *
 736  * @param dev
 737  *   Pointer to Ethernet device structure.
 738  */
 739 static void
 740 mrvl_flush_rx_queues(struct rte_eth_dev *dev)
 741 {
 742         int i;
 743
 744         MRVL_LOG(INFO, "Flushing rx queues");
 745         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 746                 int ret, num;
 747
 748                 do {
 749                         struct mrvl_rxq *q = dev->data->rx_queues[i];
 750                         struct pp2_ppio_desc descs[MRVL_PP2_RXD_MAX];
 751
 752                         num = MRVL_PP2_RXD_MAX;
 753                         ret = pp2_ppio_recv(q->priv->ppio,
 754                                             q->priv->rxq_map[q->queue_id].tc,
 755                                             q->priv->rxq_map[q->queue_id].inq,
 756                                             descs, (uint16_t *)&num);
 757                 } while (ret == 0 && num);
 758         }
 759 }
 760
 761 /**
 762  * Flush transmit shadow queues.
 763  *
 764  * @param dev
 765  *   Pointer to Ethernet device structure.
 766  */
 767 static void
 768 mrvl_flush_tx_shadow_queues(struct rte_eth_dev *dev)
 769 {
 770         int i, j;
 771         struct mrvl_txq *txq;
 772
 773         MRVL_LOG(INFO, "Flushing tx shadow queues");
 774         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 775                 txq = (struct mrvl_txq *)dev->data->tx_queues[i];
 776
 777                 for (j = 0; j < RTE_MAX_LCORE; j++) {
 778                         struct mrvl_shadow_txq *sq;
 779
 780                         if (!hifs[j])
 781                                 continue;
 782
 783                         sq = &txq->shadow_txqs[j];
 784                         mrvl_free_sent_buffers(txq->priv->ppio,
 785                                 hifs[j], j, sq, txq->queue_id, 1);
 786                         while (sq->tail != sq->head) {
 787                                 uint64_t addr = cookie_addr_high |
 788                                         sq->ent[sq->tail].buff.cookie;
 789                                 rte_pktmbuf_free(
 790                                         (struct rte_mbuf *)addr);
 791                                 sq->tail = (sq->tail + 1) &
 792                                             MRVL_PP2_TX_SHADOWQ_MASK;
 793                         }
 794                         memset(sq, 0, sizeof(*sq));
 795                 }
 796         }
 797 }
 798
 799 /**
 800  * Flush hardware bpool (buffer-pool).
 801  *
 802  * @param dev
 803  *   Pointer to Ethernet device structure.
 804  */
 805 static void
 806 mrvl_flush_bpool(struct rte_eth_dev *dev)
 807 {
 808         struct mrvl_priv *priv = dev->data->dev_private;
 809         struct pp2_hif *hif;
 810         uint32_t num;
 811         int ret;
 812         unsigned int core_id = rte_lcore_id();
 813
 814         if (core_id == LCORE_ID_ANY)
 815                 core_id = 0;
 816
 817         hif = mrvl_get_hif(priv, core_id);
 818
 819         ret = pp2_bpool_get_num_buffs(priv->bpool, &num);
 820         if (ret) {
 821                 MRVL_LOG(ERR, "Failed to get bpool buffers number");
 822                 return;
 823         }
 824
 825         while (num--) {
 826                 struct pp2_buff_inf inf;
 827                 uint64_t addr;
 828
 829                 ret = pp2_bpool_get_buff(hif, priv->bpool, &inf);
 830                 if (ret)
 831                         break;
 832
 833                 addr = cookie_addr_high | inf.cookie;
 834                 rte_pktmbuf_free((struct rte_mbuf *)addr);
 835         }
 836 }
 837
 838 /**
 839  * DPDK callback to stop the device.
 840  *
 841  * @param dev
 842  *   Pointer to Ethernet device structure.
 843  */
 844 static void
 845 mrvl_dev_stop(struct rte_eth_dev *dev)
 846 {
 847         mrvl_dev_set_link_down(dev);
 848 }
 849
 850 /**
 851  * DPDK callback to close the device.
 852  *
 853  * @param dev
 854  *   Pointer to Ethernet device structure.
 855  */
 856 static void
 857 mrvl_dev_close(struct rte_eth_dev *dev)
 858 {
 859         struct mrvl_priv *priv = dev->data->dev_private;
 860         size_t i;
 861
 862         mrvl_flush_rx_queues(dev);
 863         mrvl_flush_tx_shadow_queues(dev);
 864         mrvl_flow_deinit(dev);
 865         mrvl_mtr_deinit(dev);
 866
 867         for (i = 0; i < priv->ppio_params.inqs_params.num_tcs; ++i) {
 868                 struct pp2_ppio_tc_params *tc_params =
 869                         &priv->ppio_params.inqs_params.tcs_params[i];
 870
 871                 if (tc_params->inqs_params) {
 872                         rte_free(tc_params->inqs_params);
 873                         tc_params->inqs_params = NULL;
 874                 }
 875         }
 876
 877         if (priv->cls_tbl) {
 878                 pp2_cls_tbl_deinit(priv->cls_tbl);
 879                 priv->cls_tbl = NULL;
 880         }
 881
 882         if (priv->qos_tbl) {
 883                 pp2_cls_qos_tbl_deinit(priv->qos_tbl);
 884                 priv->qos_tbl = NULL;
 885         }
 886
 887         mrvl_flush_bpool(dev);
 888         mrvl_tm_deinit(dev);
 889
 890         if (priv->ppio) {
 891                 pp2_ppio_deinit(priv->ppio);
 892                 priv->ppio = NULL;
 893         }
 894
 895         /* policer must be released after ppio deinitialization */
 896         if (priv->default_policer) {
 897                 pp2_cls_plcr_deinit(priv->default_policer);
 898                 priv->default_policer = NULL;
 899         }
 900 }
 901
 902 /**
 903  * DPDK callback to retrieve physical link information.
 904  *
 905  * @param dev
 906  *   Pointer to Ethernet device structure.
 907  * @param wait_to_complete
 908  *   Wait for request completion (ignored).
 909  *
 910  * @return
 911  *   0 on success, negative error value otherwise.
 912  */
 913 static int
 914 mrvl_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused)
 915 {
 916         /*
 917          * TODO
 918          * once MUSDK provides necessary API use it here
 919          */
 920         struct mrvl_priv *priv = dev->data->dev_private;
 921         struct ethtool_cmd edata;
 922         struct ifreq req;
 923         int ret, fd, link_up;
 924
 925         if (!priv->ppio)
 926                 return -EPERM;
 927
 928         edata.cmd = ETHTOOL_GSET;
 929
 930         strcpy(req.ifr_name, dev->data->name);
 931         req.ifr_data = (void *)&edata;
 932
 933         fd = socket(AF_INET, SOCK_DGRAM, 0);
 934         if (fd == -1)
 935                 return -EFAULT;
 936
 937         ret = ioctl(fd, SIOCETHTOOL, &req);
 938         if (ret == -1) {
 939                 close(fd);
 940                 return -EFAULT;
 941         }
 942
 943         close(fd);
 944
 945         switch (ethtool_cmd_speed(&edata)) {
 946         case SPEED_10:
 947                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10M;
 948                 break;
 949         case SPEED_100:
 950                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_100M;
 951                 break;
 952         case SPEED_1000:
 953                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_1G;
 954                 break;
 955         case SPEED_10000:
 956                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10G;
 957                 break;
 958         default:
 959                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE;
 960         }
 961
 962         dev->data->dev_link.link_duplex = edata.duplex ? ETH_LINK_FULL_DUPLEX :
 963                                                          ETH_LINK_HALF_DUPLEX;
 964         dev->data->dev_link.link_autoneg = edata.autoneg ? ETH_LINK_AUTONEG :
 965                                                            ETH_LINK_FIXED;
 966         pp2_ppio_get_link_state(priv->ppio, &link_up);
 967         dev->data->dev_link.link_status = link_up ? ETH_LINK_UP : ETH_LINK_DOWN;
 968
 969         return 0;
 970 }
 971
 972 /**
 973  * DPDK callback to enable promiscuous mode.
 974  *
 975  * @param dev
 976  *   Pointer to Ethernet device structure.
 977  */
 978 static void
 979 mrvl_promiscuous_enable(struct rte_eth_dev *dev)
 980 {
 981         struct mrvl_priv *priv = dev->data->dev_private;
 982         int ret;
 983
 984         if (!priv->ppio)
 985                 return;
 986
 987         if (priv->isolated)
 988                 return;
 989
 990         ret = pp2_ppio_set_promisc(priv->ppio, 1);
 991         if (ret)
 992                 MRVL_LOG(ERR, "Failed to enable promiscuous mode");
 993 }
 994
 995 /**
 996  * DPDK callback to enable allmulti mode.
 997  *
 998  * @param dev
 999  *   Pointer to Ethernet device structure.
1000  */
1001 static void
1002 mrvl_allmulticast_enable(struct rte_eth_dev *dev)
1003 {
1004         struct mrvl_priv *priv = dev->data->dev_private;
1005         int ret;
1006
1007         if (!priv->ppio)
1008                 return;
1009
1010         if (priv->isolated)
1011                 return;
1012
1013         ret = pp2_ppio_set_mc_promisc(priv->ppio, 1);
1014         if (ret)
1015                 MRVL_LOG(ERR, "Failed enable all-multicast mode");
1016 }
1017
1018 /**
1019  * DPDK callback to disable promiscuous mode.
1020  *
1021  * @param dev
1022  *   Pointer to Ethernet device structure.
1023  */
1024 static void
1025 mrvl_promiscuous_disable(struct rte_eth_dev *dev)
1026 {
1027         struct mrvl_priv *priv = dev->data->dev_private;
1028         int ret;
1029
1030         if (!priv->ppio)
1031                 return;
1032
1033         ret = pp2_ppio_set_promisc(priv->ppio, 0);
1034         if (ret)
1035                 MRVL_LOG(ERR, "Failed to disable promiscuous mode");
1036 }
1037
1038 /**
1039  * DPDK callback to disable allmulticast mode.
1040  *
1041  * @param dev
1042  *   Pointer to Ethernet device structure.
1043  */
1044 static void
1045 mrvl_allmulticast_disable(struct rte_eth_dev *dev)
1046 {
1047         struct mrvl_priv *priv = dev->data->dev_private;
1048         int ret;
1049
1050         if (!priv->ppio)
1051                 return;
1052
1053         ret = pp2_ppio_set_mc_promisc(priv->ppio, 0);
1054         if (ret)
1055                 MRVL_LOG(ERR, "Failed to disable all-multicast mode");
1056 }
1057
1058 /**
1059  * DPDK callback to remove a MAC address.
1060  *
1061  * @param dev
1062  *   Pointer to Ethernet device structure.
1063  * @param index
1064  *   MAC address index.
1065  */
1066 static void
1067 mrvl_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
1068 {
1069         struct mrvl_priv *priv = dev->data->dev_private;
1070         char buf[ETHER_ADDR_FMT_SIZE];
1071         int ret;
1072
1073         if (!priv->ppio)
1074                 return;
1075
1076         if (priv->isolated)
1077                 return;
1078
1079         ret = pp2_ppio_remove_mac_addr(priv->ppio,
1080                                        dev->data->mac_addrs[index].addr_bytes);
1081         if (ret) {
1082                 ether_format_addr(buf, sizeof(buf),
1083                                   &dev->data->mac_addrs[index]);
1084                 MRVL_LOG(ERR, "Failed to remove mac %s", buf);
1085         }
1086 }
1087
1088 /**
1089  * DPDK callback to add a MAC address.
1090  *
1091  * @param dev
1092  *   Pointer to Ethernet device structure.
1093  * @param mac_addr
1094  *   MAC address to register.
1095  * @param index
1096  *   MAC address index.
1097  * @param vmdq
1098  *   VMDq pool index to associate address with (unused).
1099  *
1100  * @return
1101  *   0 on success, negative error value otherwise.
1102  */
1103 static int
1104 mrvl_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
1105                   uint32_t index, uint32_t vmdq __rte_unused)
1106 {
1107         struct mrvl_priv *priv = dev->data->dev_private;
1108         char buf[ETHER_ADDR_FMT_SIZE];
1109         int ret;
1110
1111         if (priv->isolated)
1112                 return -ENOTSUP;
1113
1114         if (index == 0)
1115                 /* For setting index 0, mrvl_mac_addr_set() should be used.*/
1116                 return -1;
1117
1118         if (!priv->ppio)
1119                 return 0;
1120
1121         /*
1122          * Maximum number of uc addresses can be tuned via kernel module mvpp2x
1123          * parameter uc_filter_max. Maximum number of mc addresses is then
1124          * MRVL_MAC_ADDRS_MAX - uc_filter_max. Currently it defaults to 4 and
1125          * 21 respectively.
1126          *
1127          * If more than uc_filter_max uc addresses were added to filter list
1128          * then NIC will switch to promiscuous mode automatically.
1129          *
1130          * If more than MRVL_MAC_ADDRS_MAX - uc_filter_max number mc addresses
1131          * were added to filter list then NIC will switch to all-multicast mode
1132          * automatically.
1133          */
1134         ret = pp2_ppio_add_mac_addr(priv->ppio, mac_addr->addr_bytes);
1135         if (ret) {
1136                 ether_format_addr(buf, sizeof(buf), mac_addr);
1137                 MRVL_LOG(ERR, "Failed to add mac %s", buf);
1138                 return -1;
1139         }
1140
1141         return 0;
1142 }
1143
1144 /**
1145  * DPDK callback to set the primary MAC address.
1146  *
1147  * @param dev
1148  *   Pointer to Ethernet device structure.
1149  * @param mac_addr
1150  *   MAC address to register.
1151  *
1152  * @return
1153  *   0 on success, negative error value otherwise.
1154  */
1155 static int
1156 mrvl_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
1157 {
1158         struct mrvl_priv *priv = dev->data->dev_private;
1159         int ret;
1160
1161         if (!priv->ppio)
1162                 return 0;
1163
1164         if (priv->isolated)
1165                 return -ENOTSUP;
1166
1167         ret = pp2_ppio_set_mac_addr(priv->ppio, mac_addr->addr_bytes);
1168         if (ret) {
1169                 char buf[ETHER_ADDR_FMT_SIZE];
1170                 ether_format_addr(buf, sizeof(buf), mac_addr);
1171                 MRVL_LOG(ERR, "Failed to set mac to %s", buf);
1172         }
1173
1174         return ret;
1175 }
1176
1177 /**
1178  * DPDK callback to get device statistics.
1179  *
1180  * @param dev
1181  *   Pointer to Ethernet device structure.
1182  * @param stats
1183  *   Stats structure output buffer.
1184  *
1185  * @return
1186  *   0 on success, negative error value otherwise.
1187  */
1188 static int
1189 mrvl_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1190 {
1191         struct mrvl_priv *priv = dev->data->dev_private;
1192         struct pp2_ppio_statistics ppio_stats;
1193         uint64_t drop_mac = 0;
1194         unsigned int i, idx, ret;
1195
1196         if (!priv->ppio)
1197                 return -EPERM;
1198
1199         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1200                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
1201                 struct pp2_ppio_inq_statistics rx_stats;
1202
1203                 if (!rxq)
1204                         continue;
1205
1206                 idx = rxq->queue_id;
1207                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
1208                         MRVL_LOG(ERR,
1209                                 "rx queue %d stats out of range (0 - %d)",
1210                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
1211                         continue;
1212                 }
1213
1214                 ret = pp2_ppio_inq_get_statistics(priv->ppio,
1215                                                   priv->rxq_map[idx].tc,
1216                                                   priv->rxq_map[idx].inq,
1217                                                   &rx_stats, 0);
1218                 if (unlikely(ret)) {
1219                         MRVL_LOG(ERR,
1220                                 "Failed to update rx queue %d stats", idx);
1221                         break;
1222                 }
1223
1224                 stats->q_ibytes[idx] = rxq->bytes_recv;
1225                 stats->q_ipackets[idx] = rx_stats.enq_desc - rxq->drop_mac;
1226                 stats->q_errors[idx] = rx_stats.drop_early +
1227                                        rx_stats.drop_fullq +
1228                                        rx_stats.drop_bm +
1229                                        rxq->drop_mac;
1230                 stats->ibytes += rxq->bytes_recv;
1231                 drop_mac += rxq->drop_mac;
1232         }
1233
1234         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1235                 struct mrvl_txq *txq = dev->data->tx_queues[i];
1236                 struct pp2_ppio_outq_statistics tx_stats;
1237
1238                 if (!txq)
1239                         continue;
1240
1241                 idx = txq->queue_id;
1242                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
1243                         MRVL_LOG(ERR,
1244                                 "tx queue %d stats out of range (0 - %d)",
1245                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
1246                 }
1247
1248                 ret = pp2_ppio_outq_get_statistics(priv->ppio, idx,
1249                                                    &tx_stats, 0);
1250                 if (unlikely(ret)) {
1251                         MRVL_LOG(ERR,
1252                                 "Failed to update tx queue %d stats", idx);
1253                         break;
1254                 }
1255
1256                 stats->q_opackets[idx] = tx_stats.deq_desc;
1257                 stats->q_obytes[idx] = txq->bytes_sent;
1258                 stats->obytes += txq->bytes_sent;
1259         }
1260
1261         ret = pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
1262         if (unlikely(ret)) {
1263                 MRVL_LOG(ERR, "Failed to update port statistics");
1264                 return ret;
1265         }
1266
1267         stats->ipackets += ppio_stats.rx_packets - drop_mac;
1268         stats->opackets += ppio_stats.tx_packets;
1269         stats->imissed += ppio_stats.rx_fullq_dropped +
1270                           ppio_stats.rx_bm_dropped +
1271                           ppio_stats.rx_early_dropped +
1272                           ppio_stats.rx_fifo_dropped +
1273                           ppio_stats.rx_cls_dropped;
1274         stats->ierrors = drop_mac;
1275
1276         return 0;
1277 }
1278
1279 /**
1280  * DPDK callback to clear device statistics.
1281  *
1282  * @param dev
1283  *   Pointer to Ethernet device structure.
1284  */
1285 static void
1286 mrvl_stats_reset(struct rte_eth_dev *dev)
1287 {
1288         struct mrvl_priv *priv = dev->data->dev_private;
1289         int i;
1290
1291         if (!priv->ppio)
1292                 return;
1293
1294         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1295                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
1296
1297                 pp2_ppio_inq_get_statistics(priv->ppio, priv->rxq_map[i].tc,
1298                                             priv->rxq_map[i].inq, NULL, 1);
1299                 rxq->bytes_recv = 0;
1300                 rxq->drop_mac = 0;
1301         }
1302
1303         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1304                 struct mrvl_txq *txq = dev->data->tx_queues[i];
1305
1306                 pp2_ppio_outq_get_statistics(priv->ppio, i, NULL, 1);
1307                 txq->bytes_sent = 0;
1308         }
1309
1310         pp2_ppio_get_statistics(priv->ppio, NULL, 1);
1311 }
1312
1313 /**
1314  * DPDK callback to get extended statistics.
1315  *
1316  * @param dev
1317  *   Pointer to Ethernet device structure.
1318  * @param stats
1319  *   Pointer to xstats table.
1320  * @param n
1321  *   Number of entries in xstats table.
1322  * @return
1323  *   Negative value on error, number of read xstats otherwise.
1324  */
1325 static int
1326 mrvl_xstats_get(struct rte_eth_dev *dev,
1327                 struct rte_eth_xstat *stats, unsigned int n)
1328 {
1329         struct mrvl_priv *priv = dev->data->dev_private;
1330         struct pp2_ppio_statistics ppio_stats;
1331         unsigned int i;
1332
1333         if (!stats)
1334                 return 0;
1335
1336         pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
1337         for (i = 0; i < n && i < RTE_DIM(mrvl_xstats_tbl); i++) {
1338                 uint64_t val;
1339
1340                 if (mrvl_xstats_tbl[i].size == sizeof(uint32_t))
1341                         val = *(uint32_t *)((uint8_t *)&ppio_stats +
1342                                             mrvl_xstats_tbl[i].offset);
1343                 else if (mrvl_xstats_tbl[i].size == sizeof(uint64_t))
1344                         val = *(uint64_t *)((uint8_t *)&ppio_stats +
1345                                             mrvl_xstats_tbl[i].offset);
1346                 else
1347                         return -EINVAL;
1348
1349                 stats[i].id = i;
1350                 stats[i].value = val;
1351         }
1352
1353         return n;
1354 }
1355
1356 /**
1357  * DPDK callback to reset extended statistics.
1358  *
1359  * @param dev
1360  *   Pointer to Ethernet device structure.
1361  */
1362 static void
1363 mrvl_xstats_reset(struct rte_eth_dev *dev)
1364 {
1365         mrvl_stats_reset(dev);
1366 }
1367
1368 /**
1369  * DPDK callback to get extended statistics names.
1370  *
1371  * @param dev (unused)
1372  *   Pointer to Ethernet device structure.
1373  * @param xstats_names
1374  *   Pointer to xstats names table.
1375  * @param size
1376  *   Size of the xstats names table.
1377  * @return
1378  *   Number of read names.
1379  */
1380 static int
1381 mrvl_xstats_get_names(struct rte_eth_dev *dev __rte_unused,
1382                       struct rte_eth_xstat_name *xstats_names,
1383                       unsigned int size)
1384 {
1385         unsigned int i;
1386
1387         if (!xstats_names)
1388                 return RTE_DIM(mrvl_xstats_tbl);
1389
1390         for (i = 0; i < size && i < RTE_DIM(mrvl_xstats_tbl); i++)
1391                 snprintf(xstats_names[i].name, RTE_ETH_XSTATS_NAME_SIZE, "%s",
1392                          mrvl_xstats_tbl[i].name);
1393
1394         return size;
1395 }
1396
1397 /**
1398  * DPDK callback to get information about the device.
1399  *
1400  * @param dev
1401  *   Pointer to Ethernet device structure (unused).
1402  * @param info
1403  *   Info structure output buffer.
1404  */
1405 static void
1406 mrvl_dev_infos_get(struct rte_eth_dev *dev __rte_unused,
1407                    struct rte_eth_dev_info *info)
1408 {
1409         info->speed_capa = ETH_LINK_SPEED_10M |
1410                            ETH_LINK_SPEED_100M |
1411                            ETH_LINK_SPEED_1G |
1412                            ETH_LINK_SPEED_10G;
1413
1414         info->max_rx_queues = MRVL_PP2_RXQ_MAX;
1415         info->max_tx_queues = MRVL_PP2_TXQ_MAX;
1416         info->max_mac_addrs = MRVL_MAC_ADDRS_MAX;
1417
1418         info->rx_desc_lim.nb_max = MRVL_PP2_RXD_MAX;
1419         info->rx_desc_lim.nb_min = MRVL_PP2_RXD_MIN;
1420         info->rx_desc_lim.nb_align = MRVL_PP2_RXD_ALIGN;
1421
1422         info->tx_desc_lim.nb_max = MRVL_PP2_TXD_MAX;
1423         info->tx_desc_lim.nb_min = MRVL_PP2_TXD_MIN;
1424         info->tx_desc_lim.nb_align = MRVL_PP2_TXD_ALIGN;
1425
1426         info->rx_offload_capa = MRVL_RX_OFFLOADS;
1427         info->rx_queue_offload_capa = MRVL_RX_OFFLOADS;
1428
1429         info->tx_offload_capa = MRVL_TX_OFFLOADS;
1430         info->tx_queue_offload_capa = MRVL_TX_OFFLOADS;
1431
1432         info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1433                                        ETH_RSS_NONFRAG_IPV4_TCP |
1434                                        ETH_RSS_NONFRAG_IPV4_UDP;
1435
1436         /* By default packets are dropped if no descriptors are available */
1437         info->default_rxconf.rx_drop_en = 1;
1438
1439         info->max_rx_pktlen = MRVL_PKT_SIZE_MAX;
1440 }
1441
1442 /**
1443  * Return supported packet types.
1444  *
1445  * @param dev
1446  *   Pointer to Ethernet device structure (unused).
1447  *
1448  * @return
1449  *   Const pointer to the table with supported packet types.
1450  */
1451 static const uint32_t *
1452 mrvl_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1453 {
1454         static const uint32_t ptypes[] = {
1455                 RTE_PTYPE_L2_ETHER,
1456                 RTE_PTYPE_L2_ETHER_VLAN,
1457                 RTE_PTYPE_L2_ETHER_QINQ,
1458                 RTE_PTYPE_L3_IPV4,
1459                 RTE_PTYPE_L3_IPV4_EXT,
1460                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
1461                 RTE_PTYPE_L3_IPV6,
1462                 RTE_PTYPE_L3_IPV6_EXT,
1463                 RTE_PTYPE_L2_ETHER_ARP,
1464                 RTE_PTYPE_L4_TCP,
1465                 RTE_PTYPE_L4_UDP
1466         };
1467
1468         return ptypes;
1469 }
1470
1471 /**
1472  * DPDK callback to get information about specific receive queue.
1473  *
1474  * @param dev
1475  *   Pointer to Ethernet device structure.
1476  * @param rx_queue_id
1477  *   Receive queue index.
1478  * @param qinfo
1479  *   Receive queue information structure.
1480  */
1481 static void mrvl_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
1482                               struct rte_eth_rxq_info *qinfo)
1483 {
1484         struct mrvl_rxq *q = dev->data->rx_queues[rx_queue_id];
1485         struct mrvl_priv *priv = dev->data->dev_private;
1486         int inq = priv->rxq_map[rx_queue_id].inq;
1487         int tc = priv->rxq_map[rx_queue_id].tc;
1488         struct pp2_ppio_tc_params *tc_params =
1489                 &priv->ppio_params.inqs_params.tcs_params[tc];
1490
1491         qinfo->mp = q->mp;
1492         qinfo->nb_desc = tc_params->inqs_params[inq].size;
1493 }
1494
1495 /**
1496  * DPDK callback to get information about specific transmit queue.
1497  *
1498  * @param dev
1499  *   Pointer to Ethernet device structure.
1500  * @param tx_queue_id
1501  *   Transmit queue index.
1502  * @param qinfo
1503  *   Transmit queue information structure.
1504  */
1505 static void mrvl_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
1506                               struct rte_eth_txq_info *qinfo)
1507 {
1508         struct mrvl_priv *priv = dev->data->dev_private;
1509         struct mrvl_txq *txq = dev->data->tx_queues[tx_queue_id];
1510
1511         qinfo->nb_desc =
1512                 priv->ppio_params.outqs_params.outqs_params[tx_queue_id].size;
1513         qinfo->conf.tx_deferred_start = txq->tx_deferred_start;
1514 }
1515
1516 /**
1517  * DPDK callback to Configure a VLAN filter.
1518  *
1519  * @param dev
1520  *   Pointer to Ethernet device structure.
1521  * @param vlan_id
1522  *   VLAN ID to filter.
1523  * @param on
1524  *   Toggle filter.
1525  *
1526  * @return
1527  *   0 on success, negative error value otherwise.
1528  */
1529 static int
1530 mrvl_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1531 {
1532         struct mrvl_priv *priv = dev->data->dev_private;
1533
1534         if (!priv->ppio)
1535                 return -EPERM;
1536
1537         if (priv->isolated)
1538                 return -ENOTSUP;
1539
1540         return on ? pp2_ppio_add_vlan(priv->ppio, vlan_id) :
1541                     pp2_ppio_remove_vlan(priv->ppio, vlan_id);
1542 }
1543
1544 /**
1545  * Release buffers to hardware bpool (buffer-pool)
1546  *
1547  * @param rxq
1548  *   Receive queue pointer.
1549  * @param num
1550  *   Number of buffers to release to bpool.
1551  *
1552  * @return
1553  *   0 on success, negative error value otherwise.
1554  */
1555 static int
1556 mrvl_fill_bpool(struct mrvl_rxq *rxq, int num)
1557 {
1558         struct buff_release_entry entries[MRVL_PP2_RXD_MAX];
1559         struct rte_mbuf *mbufs[MRVL_PP2_RXD_MAX];
1560         int i, ret;
1561         unsigned int core_id;
1562         struct pp2_hif *hif;
1563         struct pp2_bpool *bpool;
1564
1565         core_id = rte_lcore_id();
1566         if (core_id == LCORE_ID_ANY)
1567                 core_id = 0;
1568
1569         hif = mrvl_get_hif(rxq->priv, core_id);
1570         if (!hif)
1571                 return -1;
1572
1573         bpool = rxq->priv->bpool;
1574
1575         ret = rte_pktmbuf_alloc_bulk(rxq->mp, mbufs, num);
1576         if (ret)
1577                 return ret;
1578
1579         if (cookie_addr_high == MRVL_COOKIE_ADDR_INVALID)
1580                 cookie_addr_high =
1581                         (uint64_t)mbufs[0] & MRVL_COOKIE_HIGH_ADDR_MASK;
1582
1583         for (i = 0; i < num; i++) {
1584                 if (((uint64_t)mbufs[i] & MRVL_COOKIE_HIGH_ADDR_MASK)
1585                         != cookie_addr_high) {
1586                         MRVL_LOG(ERR,
1587                                 "mbuf virtual addr high 0x%lx out of range",
1588                                 (uint64_t)mbufs[i] >> 32);
1589                         goto out;
1590                 }
1591
1592                 entries[i].buff.addr =
1593                         rte_mbuf_data_iova_default(mbufs[i]);
1594                 entries[i].buff.cookie = (uint64_t)mbufs[i];
1595                 entries[i].bpool = bpool;
1596         }
1597
1598         pp2_bpool_put_buffs(hif, entries, (uint16_t *)&i);
1599         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] += i;
1600
1601         if (i != num)
1602                 goto out;
1603
1604         return 0;
1605 out:
1606         for (; i < num; i++)
1607                 rte_pktmbuf_free(mbufs[i]);
1608
1609         return -1;
1610 }
1611
1612 /**
1613  * DPDK callback to configure the receive queue.
1614  *
1615  * @param dev
1616  *   Pointer to Ethernet device structure.
1617  * @param idx
1618  *   RX queue index.
1619  * @param desc
1620  *   Number of descriptors to configure in queue.
1621  * @param socket
1622  *   NUMA socket on which memory must be allocated.
1623  * @param conf
1624  *   Thresholds parameters.
1625  * @param mp
1626  *   Memory pool for buffer allocations.
1627  *
1628  * @return
1629  *   0 on success, negative error value otherwise.
1630  */
1631 static int
1632 mrvl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1633                     unsigned int socket,
1634                     const struct rte_eth_rxconf *conf,
1635                     struct rte_mempool *mp)
1636 {
1637         struct mrvl_priv *priv = dev->data->dev_private;
1638         struct mrvl_rxq *rxq;
1639         uint32_t frame_size, buf_size = rte_pktmbuf_data_room_size(mp);
1640         uint32_t max_rx_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
1641         int ret, tc, inq;
1642         uint64_t offloads;
1643
1644         offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads;
1645
1646         if (priv->rxq_map[idx].tc == MRVL_UNKNOWN_TC) {
1647                 /*
1648                  * Unknown TC mapping, mapping will not have a correct queue.
1649                  */
1650                 MRVL_LOG(ERR, "Unknown TC mapping for queue %hu eth%hhu",
1651                         idx, priv->ppio_id);
1652                 return -EFAULT;
1653         }
1654
1655         frame_size = buf_size - RTE_PKTMBUF_HEADROOM - MRVL_PKT_EFFEC_OFFS;
1656         if (frame_size < max_rx_pkt_len) {
1657                 MRVL_LOG(WARNING,
1658                         "Mbuf size must be increased to %u bytes to hold up "
1659                         "to %u bytes of data.",
1660                         buf_size + max_rx_pkt_len - frame_size,
1661                         max_rx_pkt_len);
1662                 dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
1663                 MRVL_LOG(INFO, "Setting max rx pkt len to %u",
1664                         dev->data->dev_conf.rxmode.max_rx_pkt_len);
1665         }
1666
1667         if (dev->data->rx_queues[idx]) {
1668                 rte_free(dev->data->rx_queues[idx]);
1669                 dev->data->rx_queues[idx] = NULL;
1670         }
1671
1672         rxq = rte_zmalloc_socket("rxq", sizeof(*rxq), 0, socket);
1673         if (!rxq)
1674                 return -ENOMEM;
1675
1676         rxq->priv = priv;
1677         rxq->mp = mp;
1678         rxq->cksum_enabled = offloads & DEV_RX_OFFLOAD_IPV4_CKSUM;
1679         rxq->queue_id = idx;
1680         rxq->port_id = dev->data->port_id;
1681         mrvl_port_to_bpool_lookup[rxq->port_id] = priv->bpool;
1682
1683         tc = priv->rxq_map[rxq->queue_id].tc,
1684         inq = priv->rxq_map[rxq->queue_id].inq;
1685         priv->ppio_params.inqs_params.tcs_params[tc].inqs_params[inq].size =
1686                 desc;
1687
1688         ret = mrvl_fill_bpool(rxq, desc);
1689         if (ret) {
1690                 rte_free(rxq);
1691                 return ret;
1692         }
1693
1694         priv->bpool_init_size += desc;
1695
1696         dev->data->rx_queues[idx] = rxq;
1697
1698         return 0;
1699 }
1700
1701 /**
1702  * DPDK callback to release the receive queue.
1703  *
1704  * @param rxq
1705  *   Generic receive queue pointer.
1706  */
1707 static void
1708 mrvl_rx_queue_release(void *rxq)
1709 {
1710         struct mrvl_rxq *q = rxq;
1711         struct pp2_ppio_tc_params *tc_params;
1712         int i, num, tc, inq;
1713         struct pp2_hif *hif;
1714         unsigned int core_id = rte_lcore_id();
1715
1716         if (core_id == LCORE_ID_ANY)
1717                 core_id = 0;
1718
1719         if (!q)
1720                 return;
1721
1722         hif = mrvl_get_hif(q->priv, core_id);
1723
1724         if (!hif)
1725                 return;
1726
1727         tc = q->priv->rxq_map[q->queue_id].tc;
1728         inq = q->priv->rxq_map[q->queue_id].inq;
1729         tc_params = &q->priv->ppio_params.inqs_params.tcs_params[tc];
1730         num = tc_params->inqs_params[inq].size;
1731         for (i = 0; i < num; i++) {
1732                 struct pp2_buff_inf inf;
1733                 uint64_t addr;
1734
1735                 pp2_bpool_get_buff(hif, q->priv->bpool, &inf);
1736                 addr = cookie_addr_high | inf.cookie;
1737                 rte_pktmbuf_free((struct rte_mbuf *)addr);
1738         }
1739
1740         rte_free(q);
1741 }
1742
1743 /**
1744  * DPDK callback to configure the transmit queue.
1745  *
1746  * @param dev
1747  *   Pointer to Ethernet device structure.
1748  * @param idx
1749  *   Transmit queue index.
1750  * @param desc
1751  *   Number of descriptors to configure in the queue.
1752  * @param socket
1753  *   NUMA socket on which memory must be allocated.
1754  * @param conf
1755  *   Tx queue configuration parameters.
1756  *
1757  * @return
1758  *   0 on success, negative error value otherwise.
1759  */
1760 static int
1761 mrvl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1762                     unsigned int socket,
1763                     const struct rte_eth_txconf *conf)
1764 {
1765         struct mrvl_priv *priv = dev->data->dev_private;
1766         struct mrvl_txq *txq;
1767
1768         if (dev->data->tx_queues[idx]) {
1769                 rte_free(dev->data->tx_queues[idx]);
1770                 dev->data->tx_queues[idx] = NULL;
1771         }
1772
1773         txq = rte_zmalloc_socket("txq", sizeof(*txq), 0, socket);
1774         if (!txq)
1775                 return -ENOMEM;
1776
1777         txq->priv = priv;
1778         txq->queue_id = idx;
1779         txq->port_id = dev->data->port_id;
1780         txq->tx_deferred_start = conf->tx_deferred_start;
1781         dev->data->tx_queues[idx] = txq;
1782
1783         priv->ppio_params.outqs_params.outqs_params[idx].size = desc;
1784
1785         return 0;
1786 }
1787
1788 /**
1789  * DPDK callback to release the transmit queue.
1790  *
1791  * @param txq
1792  *   Generic transmit queue pointer.
1793  */
1794 static void
1795 mrvl_tx_queue_release(void *txq)
1796 {
1797         struct mrvl_txq *q = txq;
1798
1799         if (!q)
1800                 return;
1801
1802         rte_free(q);
1803 }
1804
1805 /**
1806  * DPDK callback to get flow control configuration.
1807  *
1808  * @param dev
1809  *  Pointer to Ethernet device structure.
1810  * @param fc_conf
1811  *  Pointer to the flow control configuration.
1812  *
1813  * @return
1814  *  0 on success, negative error value otherwise.
1815  */
1816 static int
1817 mrvl_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1818 {
1819         struct mrvl_priv *priv = dev->data->dev_private;
1820         int ret, en;
1821
1822         if (!priv)
1823                 return -EPERM;
1824
1825         ret = pp2_ppio_get_rx_pause(priv->ppio, &en);
1826         if (ret) {
1827                 MRVL_LOG(ERR, "Failed to read rx pause state");
1828                 return ret;
1829         }
1830
1831         fc_conf->mode = en ? RTE_FC_RX_PAUSE : RTE_FC_NONE;
1832
1833         return 0;
1834 }
1835
1836 /**
1837  * DPDK callback to set flow control configuration.
1838  *
1839  * @param dev
1840  *  Pointer to Ethernet device structure.
1841  * @param fc_conf
1842  *  Pointer to the flow control configuration.
1843  *
1844  * @return
1845  *  0 on success, negative error value otherwise.
1846  */
1847 static int
1848 mrvl_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1849 {
1850         struct mrvl_priv *priv = dev->data->dev_private;
1851
1852         if (!priv)
1853                 return -EPERM;
1854
1855         if (fc_conf->high_water ||
1856             fc_conf->low_water ||
1857             fc_conf->pause_time ||
1858             fc_conf->mac_ctrl_frame_fwd ||
1859             fc_conf->autoneg) {
1860                 MRVL_LOG(ERR, "Flowctrl parameter is not supported");
1861
1862                 return -EINVAL;
1863         }
1864
1865         if (fc_conf->mode == RTE_FC_NONE ||
1866             fc_conf->mode == RTE_FC_RX_PAUSE) {
1867                 int ret, en;
1868
1869                 en = fc_conf->mode == RTE_FC_NONE ? 0 : 1;
1870                 ret = pp2_ppio_set_rx_pause(priv->ppio, en);
1871                 if (ret)
1872                         MRVL_LOG(ERR,
1873                                 "Failed to change flowctrl on RX side");
1874
1875                 return ret;
1876         }
1877
1878         return 0;
1879 }
1880
1881 /**
1882  * Update RSS hash configuration
1883  *
1884  * @param dev
1885  *   Pointer to Ethernet device structure.
1886  * @param rss_conf
1887  *   Pointer to RSS configuration.
1888  *
1889  * @return
1890  *   0 on success, negative error value otherwise.
1891  */
1892 static int
1893 mrvl_rss_hash_update(struct rte_eth_dev *dev,
1894                      struct rte_eth_rss_conf *rss_conf)
1895 {
1896         struct mrvl_priv *priv = dev->data->dev_private;
1897
1898         if (priv->isolated)
1899                 return -ENOTSUP;
1900
1901         return mrvl_configure_rss(priv, rss_conf);
1902 }
1903
1904 /**
1905  * DPDK callback to get RSS hash configuration.
1906  *
1907  * @param dev
1908  *   Pointer to Ethernet device structure.
1909  * @rss_conf
1910  *   Pointer to RSS configuration.
1911  *
1912  * @return
1913  *   Always 0.
1914  */
1915 static int
1916 mrvl_rss_hash_conf_get(struct rte_eth_dev *dev,
1917                        struct rte_eth_rss_conf *rss_conf)
1918 {
1919         struct mrvl_priv *priv = dev->data->dev_private;
1920         enum pp2_ppio_hash_type hash_type =
1921                 priv->ppio_params.inqs_params.hash_type;
1922
1923         rss_conf->rss_key = NULL;
1924
1925         if (hash_type == PP2_PPIO_HASH_T_NONE)
1926                 rss_conf->rss_hf = 0;
1927         else if (hash_type == PP2_PPIO_HASH_T_2_TUPLE)
1928                 rss_conf->rss_hf = ETH_RSS_IPV4;
1929         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && priv->rss_hf_tcp)
1930                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_TCP;
1931         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && !priv->rss_hf_tcp)
1932                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_UDP;
1933
1934         return 0;
1935 }
1936
1937 /**
1938  * DPDK callback to get rte_flow callbacks.
1939  *
1940  * @param dev
1941  *   Pointer to the device structure.
1942  * @param filer_type
1943  *   Flow filter type.
1944  * @param filter_op
1945  *   Flow filter operation.
1946  * @param arg
1947  *   Pointer to pass the flow ops.
1948  *
1949  * @return
1950  *   0 on success, negative error value otherwise.
1951  */
1952 static int
1953 mrvl_eth_filter_ctrl(struct rte_eth_dev *dev __rte_unused,
1954                      enum rte_filter_type filter_type,
1955                      enum rte_filter_op filter_op, void *arg)
1956 {
1957         switch (filter_type) {
1958         case RTE_ETH_FILTER_GENERIC:
1959                 if (filter_op != RTE_ETH_FILTER_GET)
1960                         return -EINVAL;
1961                 *(const void **)arg = &mrvl_flow_ops;
1962                 return 0;
1963         default:
1964                 MRVL_LOG(WARNING, "Filter type (%d) not supported",
1965                                 filter_type);
1966                 return -EINVAL;
1967         }
1968 }
1969
1970 /**
1971  * DPDK callback to get rte_mtr callbacks.
1972  *
1973  * @param dev
1974  *   Pointer to the device structure.
1975  * @param ops
1976  *   Pointer to pass the mtr ops.
1977  *
1978  * @return
1979  *   Always 0.
1980  */
1981 static int
1982 mrvl_mtr_ops_get(struct rte_eth_dev *dev __rte_unused, void *ops)
1983 {
1984         *(const void **)ops = &mrvl_mtr_ops;
1985
1986         return 0;
1987 }
1988
1989 /**
1990  * DPDK callback to get rte_tm callbacks.
1991  *
1992  * @param dev
1993  *   Pointer to the device structure.
1994  * @param ops
1995  *   Pointer to pass the tm ops.
1996  *
1997  * @return
1998  *   Always 0.
1999  */
2000 static int
2001 mrvl_tm_ops_get(struct rte_eth_dev *dev __rte_unused, void *ops)
2002 {
2003         *(const void **)ops = &mrvl_tm_ops;
2004
2005         return 0;
2006 }
2007
2008 static const struct eth_dev_ops mrvl_ops = {
2009         .dev_configure = mrvl_dev_configure,
2010         .dev_start = mrvl_dev_start,
2011         .dev_stop = mrvl_dev_stop,
2012         .dev_set_link_up = mrvl_dev_set_link_up,
2013         .dev_set_link_down = mrvl_dev_set_link_down,
2014         .dev_close = mrvl_dev_close,
2015         .link_update = mrvl_link_update,
2016         .promiscuous_enable = mrvl_promiscuous_enable,
2017         .allmulticast_enable = mrvl_allmulticast_enable,
2018         .promiscuous_disable = mrvl_promiscuous_disable,
2019         .allmulticast_disable = mrvl_allmulticast_disable,
2020         .mac_addr_remove = mrvl_mac_addr_remove,
2021         .mac_addr_add = mrvl_mac_addr_add,
2022         .mac_addr_set = mrvl_mac_addr_set,
2023         .mtu_set = mrvl_mtu_set,
2024         .stats_get = mrvl_stats_get,
2025         .stats_reset = mrvl_stats_reset,
2026         .xstats_get = mrvl_xstats_get,
2027         .xstats_reset = mrvl_xstats_reset,
2028         .xstats_get_names = mrvl_xstats_get_names,
2029         .dev_infos_get = mrvl_dev_infos_get,
2030         .dev_supported_ptypes_get = mrvl_dev_supported_ptypes_get,
2031         .rxq_info_get = mrvl_rxq_info_get,
2032         .txq_info_get = mrvl_txq_info_get,
2033         .vlan_filter_set = mrvl_vlan_filter_set,
2034         .tx_queue_start = mrvl_tx_queue_start,
2035         .tx_queue_stop = mrvl_tx_queue_stop,
2036         .rx_queue_setup = mrvl_rx_queue_setup,
2037         .rx_queue_release = mrvl_rx_queue_release,
2038         .tx_queue_setup = mrvl_tx_queue_setup,
2039         .tx_queue_release = mrvl_tx_queue_release,
2040         .flow_ctrl_get = mrvl_flow_ctrl_get,
2041         .flow_ctrl_set = mrvl_flow_ctrl_set,
2042         .rss_hash_update = mrvl_rss_hash_update,
2043         .rss_hash_conf_get = mrvl_rss_hash_conf_get,
2044         .filter_ctrl = mrvl_eth_filter_ctrl,
2045         .mtr_ops_get = mrvl_mtr_ops_get,
2046         .tm_ops_get = mrvl_tm_ops_get,
2047 };
2048
2049 /**
2050  * Return packet type information and l3/l4 offsets.
2051  *
2052  * @param desc
2053  *   Pointer to the received packet descriptor.
2054  * @param l3_offset
2055  *   l3 packet offset.
2056  * @param l4_offset
2057  *   l4 packet offset.
2058  *
2059  * @return
2060  *   Packet type information.
2061  */
2062 static inline uint64_t
2063 mrvl_desc_to_packet_type_and_offset(struct pp2_ppio_desc *desc,
2064                                     uint8_t *l3_offset, uint8_t *l4_offset)
2065 {
2066         enum pp2_inq_l3_type l3_type;
2067         enum pp2_inq_l4_type l4_type;
2068         enum pp2_inq_vlan_tag vlan_tag;
2069         uint64_t packet_type;
2070
2071         pp2_ppio_inq_desc_get_l3_info(desc, &l3_type, l3_offset);
2072         pp2_ppio_inq_desc_get_l4_info(desc, &l4_type, l4_offset);
2073         pp2_ppio_inq_desc_get_vlan_tag(desc, &vlan_tag);
2074
2075         packet_type = RTE_PTYPE_L2_ETHER;
2076
2077         switch (vlan_tag) {
2078         case PP2_INQ_VLAN_TAG_SINGLE:
2079                 packet_type |= RTE_PTYPE_L2_ETHER_VLAN;
2080                 break;
2081         case PP2_INQ_VLAN_TAG_DOUBLE:
2082         case PP2_INQ_VLAN_TAG_TRIPLE:
2083                 packet_type |= RTE_PTYPE_L2_ETHER_QINQ;
2084                 break;
2085         default:
2086                 break;
2087         }
2088
2089         switch (l3_type) {
2090         case PP2_INQ_L3_TYPE_IPV4_NO_OPTS:
2091                 packet_type |= RTE_PTYPE_L3_IPV4;
2092                 break;
2093         case PP2_INQ_L3_TYPE_IPV4_OK:
2094                 packet_type |= RTE_PTYPE_L3_IPV4_EXT;
2095                 break;
2096         case PP2_INQ_L3_TYPE_IPV4_TTL_ZERO:
2097                 packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
2098                 break;
2099         case PP2_INQ_L3_TYPE_IPV6_NO_EXT:
2100                 packet_type |= RTE_PTYPE_L3_IPV6;
2101                 break;
2102         case PP2_INQ_L3_TYPE_IPV6_EXT:
2103                 packet_type |= RTE_PTYPE_L3_IPV6_EXT;
2104                 break;
2105         case PP2_INQ_L3_TYPE_ARP:
2106                 packet_type |= RTE_PTYPE_L2_ETHER_ARP;
2107                 /*
2108                  * In case of ARP l4_offset is set to wrong value.
2109                  * Set it to proper one so that later on mbuf->l3_len can be
2110                  * calculated subtracting l4_offset and l3_offset.
2111                  */
2112                 *l4_offset = *l3_offset + MRVL_ARP_LENGTH;
2113                 break;
2114         default:
2115                 MRVL_LOG(DEBUG, "Failed to recognise l3 packet type");
2116                 break;
2117         }
2118
2119         switch (l4_type) {
2120         case PP2_INQ_L4_TYPE_TCP:
2121                 packet_type |= RTE_PTYPE_L4_TCP;
2122                 break;
2123         case PP2_INQ_L4_TYPE_UDP:
2124                 packet_type |= RTE_PTYPE_L4_UDP;
2125                 break;
2126         default:
2127                 MRVL_LOG(DEBUG, "Failed to recognise l4 packet type");
2128                 break;
2129         }
2130
2131         return packet_type;
2132 }
2133
2134 /**
2135  * Get offload information from the received packet descriptor.
2136  *
2137  * @param desc
2138  *   Pointer to the received packet descriptor.
2139  *
2140  * @return
2141  *   Mbuf offload flags.
2142  */
2143 static inline uint64_t
2144 mrvl_desc_to_ol_flags(struct pp2_ppio_desc *desc)
2145 {
2146         uint64_t flags;
2147         enum pp2_inq_desc_status status;
2148
2149         status = pp2_ppio_inq_desc_get_l3_pkt_error(desc);
2150         if (unlikely(status != PP2_DESC_ERR_OK))
2151                 flags = PKT_RX_IP_CKSUM_BAD;
2152         else
2153                 flags = PKT_RX_IP_CKSUM_GOOD;
2154
2155         status = pp2_ppio_inq_desc_get_l4_pkt_error(desc);
2156         if (unlikely(status != PP2_DESC_ERR_OK))
2157                 flags |= PKT_RX_L4_CKSUM_BAD;
2158         else
2159                 flags |= PKT_RX_L4_CKSUM_GOOD;
2160
2161         return flags;
2162 }
2163
2164 /**
2165  * DPDK callback for receive.
2166  *
2167  * @param rxq
2168  *   Generic pointer to the receive queue.
2169  * @param rx_pkts
2170  *   Array to store received packets.
2171  * @param nb_pkts
2172  *   Maximum number of packets in array.
2173  *
2174  * @return
2175  *   Number of packets successfully received.
2176  */
2177 static uint16_t
2178 mrvl_rx_pkt_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
2179 {
2180         struct mrvl_rxq *q = rxq;
2181         struct pp2_ppio_desc descs[nb_pkts];
2182         struct pp2_bpool *bpool;
2183         int i, ret, rx_done = 0;
2184         int num;
2185         struct pp2_hif *hif;
2186         unsigned int core_id = rte_lcore_id();
2187
2188         hif = mrvl_get_hif(q->priv, core_id);
2189
2190         if (unlikely(!q->priv->ppio || !hif))
2191                 return 0;
2192
2193         bpool = q->priv->bpool;
2194
2195         ret = pp2_ppio_recv(q->priv->ppio, q->priv->rxq_map[q->queue_id].tc,
2196                             q->priv->rxq_map[q->queue_id].inq, descs, &nb_pkts);
2197         if (unlikely(ret < 0)) {
2198                 MRVL_LOG(ERR, "Failed to receive packets");
2199                 return 0;
2200         }
2201         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] -= nb_pkts;
2202
2203         for (i = 0; i < nb_pkts; i++) {
2204                 struct rte_mbuf *mbuf;
2205                 uint8_t l3_offset, l4_offset;
2206                 enum pp2_inq_desc_status status;
2207                 uint64_t addr;
2208
2209                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
2210                         struct pp2_ppio_desc *pref_desc;
2211                         u64 pref_addr;
2212
2213                         pref_desc = &descs[i + MRVL_MUSDK_PREFETCH_SHIFT];
2214                         pref_addr = cookie_addr_high |
2215                                     pp2_ppio_inq_desc_get_cookie(pref_desc);
2216                         rte_mbuf_prefetch_part1((struct rte_mbuf *)(pref_addr));
2217                         rte_mbuf_prefetch_part2((struct rte_mbuf *)(pref_addr));
2218                 }
2219
2220                 addr = cookie_addr_high |
2221                        pp2_ppio_inq_desc_get_cookie(&descs[i]);
2222                 mbuf = (struct rte_mbuf *)addr;
2223                 rte_pktmbuf_reset(mbuf);
2224
2225                 /* drop packet in case of mac, overrun or resource error */
2226                 status = pp2_ppio_inq_desc_get_l2_pkt_error(&descs[i]);
2227                 if (unlikely(status != PP2_DESC_ERR_OK)) {
2228                         struct pp2_buff_inf binf = {
2229                                 .addr = rte_mbuf_data_iova_default(mbuf),
2230                                 .cookie = (uint64_t)mbuf,
2231                         };
2232
2233                         pp2_bpool_put_buff(hif, bpool, &binf);
2234                         mrvl_port_bpool_size
2235                                 [bpool->pp2_id][bpool->id][core_id]++;
2236                         q->drop_mac++;
2237                         continue;
2238                 }
2239
2240                 mbuf->data_off += MRVL_PKT_EFFEC_OFFS;
2241                 mbuf->pkt_len = pp2_ppio_inq_desc_get_pkt_len(&descs[i]);
2242                 mbuf->data_len = mbuf->pkt_len;
2243                 mbuf->port = q->port_id;
2244                 mbuf->packet_type =
2245                         mrvl_desc_to_packet_type_and_offset(&descs[i],
2246                                                             &l3_offset,
2247                                                             &l4_offset);
2248                 mbuf->l2_len = l3_offset;
2249                 mbuf->l3_len = l4_offset - l3_offset;
2250
2251                 if (likely(q->cksum_enabled))
2252                         mbuf->ol_flags = mrvl_desc_to_ol_flags(&descs[i]);
2253
2254                 rx_pkts[rx_done++] = mbuf;
2255                 q->bytes_recv += mbuf->pkt_len;
2256         }
2257
2258         if (rte_spinlock_trylock(&q->priv->lock) == 1) {
2259                 num = mrvl_get_bpool_size(bpool->pp2_id, bpool->id);
2260
2261                 if (unlikely(num <= q->priv->bpool_min_size ||
2262                              (!rx_done && num < q->priv->bpool_init_size))) {
2263                         ret = mrvl_fill_bpool(q, MRVL_BURST_SIZE);
2264                         if (ret)
2265                                 MRVL_LOG(ERR, "Failed to fill bpool");
2266                 } else if (unlikely(num > q->priv->bpool_max_size)) {
2267                         int i;
2268                         int pkt_to_remove = num - q->priv->bpool_init_size;
2269                         struct rte_mbuf *mbuf;
2270                         struct pp2_buff_inf buff;
2271
2272                         MRVL_LOG(DEBUG,
2273                                 "port-%d:%d: bpool %d oversize - remove %d buffers (pool size: %d -> %d)",
2274                                 bpool->pp2_id, q->priv->ppio->port_id,
2275                                 bpool->id, pkt_to_remove, num,
2276                                 q->priv->bpool_init_size);
2277
2278                         for (i = 0; i < pkt_to_remove; i++) {
2279                                 ret = pp2_bpool_get_buff(hif, bpool, &buff);
2280                                 if (ret)
2281                                         break;
2282                                 mbuf = (struct rte_mbuf *)
2283                                         (cookie_addr_high | buff.cookie);
2284                                 rte_pktmbuf_free(mbuf);
2285                         }
2286                         mrvl_port_bpool_size
2287                                 [bpool->pp2_id][bpool->id][core_id] -= i;
2288                 }
2289                 rte_spinlock_unlock(&q->priv->lock);
2290         }
2291
2292         return rx_done;
2293 }
2294
2295 /**
2296  * Prepare offload information.
2297  *
2298  * @param ol_flags
2299  *   Offload flags.
2300  * @param packet_type
2301  *   Packet type bitfield.
2302  * @param l3_type
2303  *   Pointer to the pp2_ouq_l3_type structure.
2304  * @param l4_type
2305  *   Pointer to the pp2_outq_l4_type structure.
2306  * @param gen_l3_cksum
2307  *   Will be set to 1 in case l3 checksum is computed.
2308  * @param l4_cksum
2309  *   Will be set to 1 in case l4 checksum is computed.
2310  *
2311  * @return
2312  *   0 on success, negative error value otherwise.
2313  */
2314 static inline int
2315 mrvl_prepare_proto_info(uint64_t ol_flags, uint32_t packet_type,
2316                         enum pp2_outq_l3_type *l3_type,
2317                         enum pp2_outq_l4_type *l4_type,
2318                         int *gen_l3_cksum,
2319                         int *gen_l4_cksum)
2320 {
2321         /*
2322          * Based on ol_flags prepare information
2323          * for pp2_ppio_outq_desc_set_proto_info() which setups descriptor
2324          * for offloading.
2325          */
2326         if (ol_flags & PKT_TX_IPV4) {
2327                 *l3_type = PP2_OUTQ_L3_TYPE_IPV4;
2328                 *gen_l3_cksum = ol_flags & PKT_TX_IP_CKSUM ? 1 : 0;
2329         } else if (ol_flags & PKT_TX_IPV6) {
2330                 *l3_type = PP2_OUTQ_L3_TYPE_IPV6;
2331                 /* no checksum for ipv6 header */
2332                 *gen_l3_cksum = 0;
2333         } else {
2334                 /* if something different then stop processing */
2335                 return -1;
2336         }
2337
2338         ol_flags &= PKT_TX_L4_MASK;
2339         if ((packet_type & RTE_PTYPE_L4_TCP) &&
2340             ol_flags == PKT_TX_TCP_CKSUM) {
2341                 *l4_type = PP2_OUTQ_L4_TYPE_TCP;
2342                 *gen_l4_cksum = 1;
2343         } else if ((packet_type & RTE_PTYPE_L4_UDP) &&
2344                    ol_flags == PKT_TX_UDP_CKSUM) {
2345                 *l4_type = PP2_OUTQ_L4_TYPE_UDP;
2346                 *gen_l4_cksum = 1;
2347         } else {
2348                 *l4_type = PP2_OUTQ_L4_TYPE_OTHER;
2349                 /* no checksum for other type */
2350                 *gen_l4_cksum = 0;
2351         }
2352
2353         return 0;
2354 }
2355
2356 /**
2357  * Release already sent buffers to bpool (buffer-pool).
2358  *
2359  * @param ppio
2360  *   Pointer to the port structure.
2361  * @param hif
2362  *   Pointer to the MUSDK hardware interface.
2363  * @param sq
2364  *   Pointer to the shadow queue.
2365  * @param qid
2366  *   Queue id number.
2367  * @param force
2368  *   Force releasing packets.
2369  */
2370 static inline void
2371 mrvl_free_sent_buffers(struct pp2_ppio *ppio, struct pp2_hif *hif,
2372                        unsigned int core_id, struct mrvl_shadow_txq *sq,
2373                        int qid, int force)
2374 {
2375         struct buff_release_entry *entry;
2376         uint16_t nb_done = 0, num = 0, skip_bufs = 0;
2377         int i;
2378
2379         pp2_ppio_get_num_outq_done(ppio, hif, qid, &nb_done);
2380
2381         sq->num_to_release += nb_done;
2382
2383         if (likely(!force &&
2384                    sq->num_to_release < MRVL_PP2_BUF_RELEASE_BURST_SIZE))
2385                 return;
2386
2387         nb_done = sq->num_to_release;
2388         sq->num_to_release = 0;
2389
2390         for (i = 0; i < nb_done; i++) {
2391                 entry = &sq->ent[sq->tail + num];
2392                 if (unlikely(!entry->buff.addr)) {
2393                         MRVL_LOG(ERR,
2394                                 "Shadow memory @%d: cookie(%lx), pa(%lx)!",
2395                                 sq->tail, (u64)entry->buff.cookie,
2396                                 (u64)entry->buff.addr);
2397                         skip_bufs = 1;
2398                         goto skip;
2399                 }
2400
2401                 if (unlikely(!entry->bpool)) {
2402                         struct rte_mbuf *mbuf;
2403
2404                         mbuf = (struct rte_mbuf *)
2405                                (cookie_addr_high | entry->buff.cookie);
2406                         rte_pktmbuf_free(mbuf);
2407                         skip_bufs = 1;
2408                         goto skip;
2409                 }
2410
2411                 mrvl_port_bpool_size
2412                         [entry->bpool->pp2_id][entry->bpool->id][core_id]++;
2413                 num++;
2414                 if (unlikely(sq->tail + num == MRVL_PP2_TX_SHADOWQ_SIZE))
2415                         goto skip;
2416                 continue;
2417 skip:
2418                 if (likely(num))
2419                         pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
2420                 num += skip_bufs;
2421                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
2422                 sq->size -= num;
2423                 num = 0;
2424                 skip_bufs = 0;
2425         }
2426
2427         if (likely(num)) {
2428                 pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
2429                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
2430                 sq->size -= num;
2431         }
2432 }
2433
2434 /**
2435  * DPDK callback for transmit.
2436  *
2437  * @param txq
2438  *   Generic pointer transmit queue.
2439  * @param tx_pkts
2440  *   Packets to transmit.
2441  * @param nb_pkts
2442  *   Number of packets in array.
2443  *
2444  * @return
2445  *   Number of packets successfully transmitted.
2446  */
2447 static uint16_t
2448 mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
2449 {
2450         struct mrvl_txq *q = txq;
2451         struct mrvl_shadow_txq *sq;
2452         struct pp2_hif *hif;
2453         struct pp2_ppio_desc descs[nb_pkts];
2454         unsigned int core_id = rte_lcore_id();
2455         int i, ret, bytes_sent = 0;
2456         uint16_t num, sq_free_size;
2457         uint64_t addr;
2458
2459         hif = mrvl_get_hif(q->priv, core_id);
2460         sq = &q->shadow_txqs[core_id];
2461
2462         if (unlikely(!q->priv->ppio || !hif))
2463                 return 0;
2464
2465         if (sq->size)
2466                 mrvl_free_sent_buffers(q->priv->ppio, hif, core_id,
2467                                        sq, q->queue_id, 0);
2468
2469         sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1;
2470         if (unlikely(nb_pkts > sq_free_size)) {
2471                 MRVL_LOG(DEBUG,
2472                         "No room in shadow queue for %d packets! %d packets will be sent.",
2473                         nb_pkts, sq_free_size);
2474                 nb_pkts = sq_free_size;
2475         }
2476
2477         for (i = 0; i < nb_pkts; i++) {
2478                 struct rte_mbuf *mbuf = tx_pkts[i];
2479                 int gen_l3_cksum, gen_l4_cksum;
2480                 enum pp2_outq_l3_type l3_type;
2481                 enum pp2_outq_l4_type l4_type;
2482
2483                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
2484                         struct rte_mbuf *pref_pkt_hdr;
2485
2486                         pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT];
2487                         rte_mbuf_prefetch_part1(pref_pkt_hdr);
2488                         rte_mbuf_prefetch_part2(pref_pkt_hdr);
2489                 }
2490
2491                 mrvl_fill_shadowq(sq, mbuf);
2492                 mrvl_fill_desc(&descs[i], mbuf);
2493
2494                 bytes_sent += rte_pktmbuf_pkt_len(mbuf);
2495                 /*
2496                  * in case unsupported ol_flags were passed
2497                  * do not update descriptor offload information
2498                  */
2499                 ret = mrvl_prepare_proto_info(mbuf->ol_flags, mbuf->packet_type,
2500                                               &l3_type, &l4_type, &gen_l3_cksum,
2501                                               &gen_l4_cksum);
2502                 if (unlikely(ret))
2503                         continue;
2504
2505                 pp2_ppio_outq_desc_set_proto_info(&descs[i], l3_type, l4_type,
2506                                                   mbuf->l2_len,
2507                                                   mbuf->l2_len + mbuf->l3_len,
2508                                                   gen_l3_cksum, gen_l4_cksum);
2509         }
2510
2511         num = nb_pkts;
2512         pp2_ppio_send(q->priv->ppio, hif, q->queue_id, descs, &nb_pkts);
2513         /* number of packets that were not sent */
2514         if (unlikely(num > nb_pkts)) {
2515                 for (i = nb_pkts; i < num; i++) {
2516                         sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) &
2517                                 MRVL_PP2_TX_SHADOWQ_MASK;
2518                         addr = cookie_addr_high | sq->ent[sq->head].buff.cookie;
2519                         bytes_sent -=
2520                                 rte_pktmbuf_pkt_len((struct rte_mbuf *)addr);
2521                 }
2522                 sq->size -= num - nb_pkts;
2523         }
2524
2525         q->bytes_sent += bytes_sent;
2526
2527         return nb_pkts;
2528 }
2529
2530 /** DPDK callback for S/G transmit.
2531  *
2532  * @param txq
2533  *   Generic pointer transmit queue.
2534  * @param tx_pkts
2535  *   Packets to transmit.
2536  * @param nb_pkts
2537  *   Number of packets in array.
2538  *
2539  * @return
2540  *   Number of packets successfully transmitted.
2541  */
2542 static uint16_t
2543 mrvl_tx_sg_pkt_burst(void *txq, struct rte_mbuf **tx_pkts,
2544                      uint16_t nb_pkts)
2545 {
2546         struct mrvl_txq *q = txq;
2547         struct mrvl_shadow_txq *sq;
2548         struct pp2_hif *hif;
2549         struct pp2_ppio_desc descs[nb_pkts * PP2_PPIO_DESC_NUM_FRAGS];
2550         struct pp2_ppio_sg_pkts pkts;
2551         uint8_t frags[nb_pkts];
2552         unsigned int core_id = rte_lcore_id();
2553         int i, j, ret, bytes_sent = 0;
2554         int tail, tail_first;
2555         uint16_t num, sq_free_size;
2556         uint16_t nb_segs, total_descs = 0;
2557         uint64_t addr;
2558
2559         hif = mrvl_get_hif(q->priv, core_id);
2560         sq = &q->shadow_txqs[core_id];
2561         pkts.frags = frags;
2562         pkts.num = 0;
2563
2564         if (unlikely(!q->priv->ppio || !hif))
2565                 return 0;
2566
2567         if (sq->size)
2568                 mrvl_free_sent_buffers(q->priv->ppio, hif, core_id,
2569                                        sq, q->queue_id, 0);
2570
2571         /* Save shadow queue free size */
2572         sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1;
2573
2574         tail = 0;
2575         for (i = 0; i < nb_pkts; i++) {
2576                 struct rte_mbuf *mbuf = tx_pkts[i];
2577                 struct rte_mbuf *seg = NULL;
2578                 int gen_l3_cksum, gen_l4_cksum;
2579                 enum pp2_outq_l3_type l3_type;
2580                 enum pp2_outq_l4_type l4_type;
2581
2582                 nb_segs = mbuf->nb_segs;
2583                 tail_first = tail;
2584                 total_descs += nb_segs;
2585
2586                 /*
2587                  * Check if total_descs does not exceed
2588                  * shadow queue free size
2589                  */
2590                 if (unlikely(total_descs > sq_free_size)) {
2591                         total_descs -= nb_segs;
2592                         RTE_LOG(DEBUG, PMD,
2593                                 "No room in shadow queue for %d packets! "
2594                                 "%d packets will be sent.\n",
2595                                 nb_pkts, i);
2596                         break;
2597                 }
2598
2599                 /* Check if nb_segs does not exceed the max nb of desc per
2600                  * fragmented packet
2601                  */
2602                 if (nb_segs > PP2_PPIO_DESC_NUM_FRAGS) {
2603                         total_descs -= nb_segs;
2604                         RTE_LOG(ERR, PMD,
2605                                 "Too many segments. Packet won't be sent.\n");
2606                         break;
2607                 }
2608
2609                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
2610                         struct rte_mbuf *pref_pkt_hdr;
2611
2612                         pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT];
2613                         rte_mbuf_prefetch_part1(pref_pkt_hdr);
2614                         rte_mbuf_prefetch_part2(pref_pkt_hdr);
2615                 }
2616
2617                 pkts.frags[pkts.num] = nb_segs;
2618                 pkts.num++;
2619
2620                 seg = mbuf;
2621                 for (j = 0; j < nb_segs - 1; j++) {
2622                         /* For the subsequent segments, set shadow queue
2623                          * buffer to NULL
2624                          */
2625                         mrvl_fill_shadowq(sq, NULL);
2626                         mrvl_fill_desc(&descs[tail], seg);
2627
2628                         tail++;
2629                         seg = seg->next;
2630                 }
2631                 /* Put first mbuf info in last shadow queue entry */
2632                 mrvl_fill_shadowq(sq, mbuf);
2633                 /* Update descriptor with last segment */
2634                 mrvl_fill_desc(&descs[tail++], seg);
2635
2636                 bytes_sent += rte_pktmbuf_pkt_len(mbuf);
2637                 /* In case unsupported ol_flags were passed
2638                  * do not update descriptor offload information
2639                  */
2640                 ret = mrvl_prepare_proto_info(mbuf->ol_flags, mbuf->packet_type,
2641                                               &l3_type, &l4_type, &gen_l3_cksum,
2642                                               &gen_l4_cksum);
2643                 if (unlikely(ret))
2644                         continue;
2645
2646                 pp2_ppio_outq_desc_set_proto_info(&descs[tail_first], l3_type,
2647                                                   l4_type, mbuf->l2_len,
2648                                                   mbuf->l2_len + mbuf->l3_len,
2649                                                   gen_l3_cksum, gen_l4_cksum);
2650         }
2651
2652         num = total_descs;
2653         pp2_ppio_send_sg(q->priv->ppio, hif, q->queue_id, descs,
2654                          &total_descs, &pkts);
2655         /* number of packets that were not sent */
2656         if (unlikely(num > total_descs)) {
2657                 for (i = total_descs; i < num; i++) {
2658                         sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) &
2659                                 MRVL_PP2_TX_SHADOWQ_MASK;
2660
2661                         addr = sq->ent[sq->head].buff.cookie;
2662                         if (addr)
2663                                 bytes_sent -=
2664                                         rte_pktmbuf_pkt_len((struct rte_mbuf *)
2665                                                 (cookie_addr_high | addr));
2666                 }
2667                 sq->size -= num - total_descs;
2668                 nb_pkts = pkts.num;
2669         }
2670
2671         q->bytes_sent += bytes_sent;
2672
2673         return nb_pkts;
2674 }
2675
2676 /**
2677  * Initialize packet processor.
2678  *
2679  * @return
2680  *   0 on success, negative error value otherwise.
2681  */
2682 static int
2683 mrvl_init_pp2(void)
2684 {
2685         struct pp2_init_params init_params;
2686
2687         memset(&init_params, 0, sizeof(init_params));
2688         init_params.hif_reserved_map = MRVL_MUSDK_HIFS_RESERVED;
2689         init_params.bm_pool_reserved_map = MRVL_MUSDK_BPOOLS_RESERVED;
2690         init_params.rss_tbl_reserved_map = MRVL_MUSDK_RSS_RESERVED;
2691
2692         return pp2_init(&init_params);
2693 }
2694
2695 /**
2696  * Deinitialize packet processor.
2697  *
2698  * @return
2699  *   0 on success, negative error value otherwise.
2700  */
2701 static void
2702 mrvl_deinit_pp2(void)
2703 {
2704         pp2_deinit();
2705 }
2706
2707 /**
2708  * Create private device structure.
2709  *
2710  * @param dev_name
2711  *   Pointer to the port name passed in the initialization parameters.
2712  *
2713  * @return
2714  *   Pointer to the newly allocated private device structure.
2715  */
2716 static struct mrvl_priv *
2717 mrvl_priv_create(const char *dev_name)
2718 {
2719         struct pp2_bpool_params bpool_params;
2720         char match[MRVL_MATCH_LEN];
2721         struct mrvl_priv *priv;
2722         int ret, bpool_bit;
2723
2724         priv = rte_zmalloc_socket(dev_name, sizeof(*priv), 0, rte_socket_id());
2725         if (!priv)
2726                 return NULL;
2727
2728         ret = pp2_netdev_get_ppio_info((char *)(uintptr_t)dev_name,
2729                                        &priv->pp_id, &priv->ppio_id);
2730         if (ret)
2731                 goto out_free_priv;
2732
2733         bpool_bit = mrvl_reserve_bit(&used_bpools[priv->pp_id],
2734                                      PP2_BPOOL_NUM_POOLS);
2735         if (bpool_bit < 0)
2736                 goto out_free_priv;
2737         priv->bpool_bit = bpool_bit;
2738
2739         snprintf(match, sizeof(match), "pool-%d:%d", priv->pp_id,
2740                  priv->bpool_bit);
2741         memset(&bpool_params, 0, sizeof(bpool_params));
2742         bpool_params.match = match;
2743         bpool_params.buff_len = MRVL_PKT_SIZE_MAX + MRVL_PKT_EFFEC_OFFS;
2744         ret = pp2_bpool_init(&bpool_params, &priv->bpool);
2745         if (ret)
2746                 goto out_clear_bpool_bit;
2747
2748         priv->ppio_params.type = PP2_PPIO_T_NIC;
2749         rte_spinlock_init(&priv->lock);
2750
2751         return priv;
2752 out_clear_bpool_bit:
2753         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
2754 out_free_priv:
2755         rte_free(priv);
2756         return NULL;
2757 }
2758
2759 /**
2760  * Create device representing Ethernet port.
2761  *
2762  * @param name
2763  *   Pointer to the port's name.
2764  *
2765  * @return
2766  *   0 on success, negative error value otherwise.
2767  */
2768 static int
2769 mrvl_eth_dev_create(struct rte_vdev_device *vdev, const char *name)
2770 {
2771         int ret, fd = socket(AF_INET, SOCK_DGRAM, 0);
2772         struct rte_eth_dev *eth_dev;
2773         struct mrvl_priv *priv;
2774         struct ifreq req;
2775
2776         eth_dev = rte_eth_dev_allocate(name);
2777         if (!eth_dev)
2778                 return -ENOMEM;
2779
2780         priv = mrvl_priv_create(name);
2781         if (!priv) {
2782                 ret = -ENOMEM;
2783                 goto out_free;
2784         }
2785         eth_dev->data->dev_private = priv;
2786
2787         eth_dev->data->mac_addrs =
2788                 rte_zmalloc("mac_addrs",
2789                             ETHER_ADDR_LEN * MRVL_MAC_ADDRS_MAX, 0);
2790         if (!eth_dev->data->mac_addrs) {
2791                 MRVL_LOG(ERR, "Failed to allocate space for eth addrs");
2792                 ret = -ENOMEM;
2793                 goto out_free;
2794         }
2795
2796         memset(&req, 0, sizeof(req));
2797         strcpy(req.ifr_name, name);
2798         ret = ioctl(fd, SIOCGIFHWADDR, &req);
2799         if (ret)
2800                 goto out_free;
2801
2802         memcpy(eth_dev->data->mac_addrs[0].addr_bytes,
2803                req.ifr_addr.sa_data, ETHER_ADDR_LEN);
2804
2805         eth_dev->data->kdrv = RTE_KDRV_NONE;
2806         eth_dev->device = &vdev->device;
2807         eth_dev->rx_pkt_burst = mrvl_rx_pkt_burst;
2808         mrvl_set_tx_function(eth_dev);
2809         eth_dev->dev_ops = &mrvl_ops;
2810
2811         rte_eth_dev_probing_finish(eth_dev);
2812         return 0;
2813 out_free:
2814         rte_eth_dev_release_port(eth_dev);
2815
2816         return ret;
2817 }
2818
2819 /**
2820  * Cleanup previously created device representing Ethernet port.
2821  *
2822  * @param name
2823  *   Pointer to the port name.
2824  */
2825 static void
2826 mrvl_eth_dev_destroy(const char *name)
2827 {
2828         struct rte_eth_dev *eth_dev;
2829         struct mrvl_priv *priv;
2830
2831         eth_dev = rte_eth_dev_allocated(name);
2832         if (!eth_dev)
2833                 return;
2834
2835         priv = eth_dev->data->dev_private;
2836         pp2_bpool_deinit(priv->bpool);
2837         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
2838         rte_eth_dev_release_port(eth_dev);
2839 }
2840
2841 /**
2842  * Callback used by rte_kvargs_process() during argument parsing.
2843  *
2844  * @param key
2845  *   Pointer to the parsed key (unused).
2846  * @param value
2847  *   Pointer to the parsed value.
2848  * @param extra_args
2849  *   Pointer to the extra arguments which contains address of the
2850  *   table of pointers to parsed interface names.
2851  *
2852  * @return
2853  *   Always 0.
2854  */
2855 static int
2856 mrvl_get_ifnames(const char *key __rte_unused, const char *value,
2857                  void *extra_args)
2858 {
2859         struct mrvl_ifnames *ifnames = extra_args;
2860
2861         ifnames->names[ifnames->idx++] = value;
2862
2863         return 0;
2864 }
2865
2866 /**
2867  * Deinitialize per-lcore MUSDK hardware interfaces (hifs).
2868  */
2869 static void
2870 mrvl_deinit_hifs(void)
2871 {
2872         int i;
2873
2874         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++) {
2875                 if (hifs[i])
2876                         pp2_hif_deinit(hifs[i]);
2877         }
2878         used_hifs = MRVL_MUSDK_HIFS_RESERVED;
2879         memset(hifs, 0, sizeof(hifs));
2880 }
2881
2882 /**
2883  * DPDK callback to register the virtual device.
2884  *
2885  * @param vdev
2886  *   Pointer to the virtual device.
2887  *
2888  * @return
2889  *   0 on success, negative error value otherwise.
2890  */
2891 static int
2892 rte_pmd_mrvl_probe(struct rte_vdev_device *vdev)
2893 {
2894         struct rte_kvargs *kvlist;
2895         struct mrvl_ifnames ifnames;
2896         int ret = -EINVAL;
2897         uint32_t i, ifnum, cfgnum;
2898         const char *params;
2899
2900         params = rte_vdev_device_args(vdev);
2901         if (!params)
2902                 return -EINVAL;
2903
2904         kvlist = rte_kvargs_parse(params, valid_args);
2905         if (!kvlist)
2906                 return -EINVAL;
2907
2908         ifnum = rte_kvargs_count(kvlist, MRVL_IFACE_NAME_ARG);
2909         if (ifnum > RTE_DIM(ifnames.names))
2910                 goto out_free_kvlist;
2911
2912         ifnames.idx = 0;
2913         rte_kvargs_process(kvlist, MRVL_IFACE_NAME_ARG,
2914                            mrvl_get_ifnames, &ifnames);
2915
2916
2917         /*
2918          * The below system initialization should be done only once,
2919          * on the first provided configuration file
2920          */
2921         if (!mrvl_qos_cfg) {
2922                 cfgnum = rte_kvargs_count(kvlist, MRVL_CFG_ARG);
2923                 MRVL_LOG(INFO, "Parsing config file!");
2924                 if (cfgnum > 1) {
2925                         MRVL_LOG(ERR, "Cannot handle more than one config file!");
2926                         goto out_free_kvlist;
2927                 } else if (cfgnum == 1) {
2928                         rte_kvargs_process(kvlist, MRVL_CFG_ARG,
2929                                            mrvl_get_qoscfg, &mrvl_qos_cfg);
2930                 }
2931         }
2932
2933         if (mrvl_dev_num)
2934                 goto init_devices;
2935
2936         MRVL_LOG(INFO, "Perform MUSDK initializations");
2937
2938         ret = rte_mvep_init(MVEP_MOD_T_PP2, kvlist);
2939         if (ret)
2940                 goto out_free_kvlist;
2941
2942         ret = mrvl_init_pp2();
2943         if (ret) {
2944                 MRVL_LOG(ERR, "Failed to init PP!");
2945                 rte_mvep_deinit(MVEP_MOD_T_PP2);
2946                 goto out_free_kvlist;
2947         }
2948
2949         memset(mrvl_port_bpool_size, 0, sizeof(mrvl_port_bpool_size));
2950         memset(mrvl_port_to_bpool_lookup, 0, sizeof(mrvl_port_to_bpool_lookup));
2951
2952         mrvl_lcore_first = RTE_MAX_LCORE;
2953         mrvl_lcore_last = 0;
2954
2955 init_devices:
2956         for (i = 0; i < ifnum; i++) {
2957                 MRVL_LOG(INFO, "Creating %s", ifnames.names[i]);
2958                 ret = mrvl_eth_dev_create(vdev, ifnames.names[i]);
2959                 if (ret)
2960                         goto out_cleanup;
2961         }
2962         mrvl_dev_num += ifnum;
2963
2964         rte_kvargs_free(kvlist);
2965
2966         return 0;
2967 out_cleanup:
2968         for (; i > 0; i--)
2969                 mrvl_eth_dev_destroy(ifnames.names[i]);
2970
2971         if (mrvl_dev_num == 0) {
2972                 mrvl_deinit_pp2();
2973                 rte_mvep_deinit(MVEP_MOD_T_PP2);
2974         }
2975 out_free_kvlist:
2976         rte_kvargs_free(kvlist);
2977
2978         return ret;
2979 }
2980
2981 /**
2982  * DPDK callback to remove virtual device.
2983  *
2984  * @param vdev
2985  *   Pointer to the removed virtual device.
2986  *
2987  * @return
2988  *   0 on success, negative error value otherwise.
2989  */
2990 static int
2991 rte_pmd_mrvl_remove(struct rte_vdev_device *vdev)
2992 {
2993         int i;
2994         const char *name;
2995
2996         name = rte_vdev_device_name(vdev);
2997         if (!name)
2998                 return -EINVAL;
2999
3000         MRVL_LOG(INFO, "Removing %s", name);
3001
3002         RTE_ETH_FOREACH_DEV(i) { /* FIXME: removing all devices! */
3003                 char ifname[RTE_ETH_NAME_MAX_LEN];
3004
3005                 rte_eth_dev_get_name_by_port(i, ifname);
3006                 mrvl_eth_dev_destroy(ifname);
3007                 mrvl_dev_num--;
3008         }
3009
3010         if (mrvl_dev_num == 0) {
3011                 MRVL_LOG(INFO, "Perform MUSDK deinit");
3012                 mrvl_deinit_hifs();
3013                 mrvl_deinit_pp2();
3014                 rte_mvep_deinit(MVEP_MOD_T_PP2);
3015         }
3016
3017         return 0;
3018 }
3019
3020 static struct rte_vdev_driver pmd_mrvl_drv = {
3021         .probe = rte_pmd_mrvl_probe,
3022         .remove = rte_pmd_mrvl_remove,
3023 };
3024
3025 RTE_PMD_REGISTER_VDEV(net_mvpp2, pmd_mrvl_drv);
3026 RTE_PMD_REGISTER_ALIAS(net_mvpp2, eth_mvpp2);
3027
3028 RTE_INIT(mrvl_init_log)
3029 {
3030         mrvl_logtype = rte_log_register("pmd.net.mvpp2");
3031         if (mrvl_logtype >= 0)
3032                 rte_log_set_level(mrvl_logtype, RTE_LOG_NOTICE);
3033 }