lib/librte_ethdev/rte_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <sys/types.h>
   6 #include <sys/queue.h>
   7 #include <ctype.h>
   8 #include <stdio.h>
   9 #include <stdlib.h>
  10 #include <string.h>
  11 #include <stdarg.h>
  12 #include <errno.h>
  13 #include <stdbool.h>
  14 #include <stdint.h>
  15 #include <inttypes.h>
  16 #include <netinet/in.h>
  17
  18 #include <rte_byteorder.h>
  19 #include <rte_log.h>
  20 #include <rte_debug.h>
  21 #include <rte_interrupts.h>
  22 #include <rte_memory.h>
  23 #include <rte_memcpy.h>
  24 #include <rte_memzone.h>
  25 #include <rte_launch.h>
  26 #include <rte_eal.h>
  27 #include <rte_per_lcore.h>
  28 #include <rte_lcore.h>
  29 #include <rte_atomic.h>
  30 #include <rte_branch_prediction.h>
  31 #include <rte_common.h>
  32 #include <rte_mempool.h>
  33 #include <rte_malloc.h>
  34 #include <rte_mbuf.h>
  35 #include <rte_errno.h>
  36 #include <rte_spinlock.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_kvargs.h>
  39 #include <rte_class.h>
  40
  41 #include "rte_ether.h"
  42 #include "rte_ethdev.h"
  43 #include "rte_ethdev_driver.h"
  44 #include "ethdev_profile.h"
  45 #include "ethdev_private.h"
  46
  47 int rte_eth_dev_logtype;
  48
  49 static const char *MZ_RTE_ETH_DEV_DATA = "rte_eth_dev_data";
  50 struct rte_eth_dev rte_eth_devices[RTE_MAX_ETHPORTS];
  51 static uint16_t eth_dev_last_created_port;
  52
  53 /* spinlock for eth device callbacks */
  54 static rte_spinlock_t rte_eth_dev_cb_lock = RTE_SPINLOCK_INITIALIZER;
  55
  56 /* spinlock for add/remove rx callbacks */
  57 static rte_spinlock_t rte_eth_rx_cb_lock = RTE_SPINLOCK_INITIALIZER;
  58
  59 /* spinlock for add/remove tx callbacks */
  60 static rte_spinlock_t rte_eth_tx_cb_lock = RTE_SPINLOCK_INITIALIZER;
  61
  62 /* spinlock for shared data allocation */
  63 static rte_spinlock_t rte_eth_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
  64
  65 /* store statistics names and its offset in stats structure  */
  66 struct rte_eth_xstats_name_off {
  67         char name[RTE_ETH_XSTATS_NAME_SIZE];
  68         unsigned offset;
  69 };
  70
  71 /* Shared memory between primary and secondary processes. */
  72 static struct {
  73         uint64_t next_owner_id;
  74         rte_spinlock_t ownership_lock;
  75         struct rte_eth_dev_data data[RTE_MAX_ETHPORTS];
  76 } *rte_eth_dev_shared_data;
  77
  78 static const struct rte_eth_xstats_name_off rte_stats_strings[] = {
  79         {"rx_good_packets", offsetof(struct rte_eth_stats, ipackets)},
  80         {"tx_good_packets", offsetof(struct rte_eth_stats, opackets)},
  81         {"rx_good_bytes", offsetof(struct rte_eth_stats, ibytes)},
  82         {"tx_good_bytes", offsetof(struct rte_eth_stats, obytes)},
  83         {"rx_missed_errors", offsetof(struct rte_eth_stats, imissed)},
  84         {"rx_errors", offsetof(struct rte_eth_stats, ierrors)},
  85         {"tx_errors", offsetof(struct rte_eth_stats, oerrors)},
  86         {"rx_mbuf_allocation_errors", offsetof(struct rte_eth_stats,
  87                 rx_nombuf)},
  88 };
  89
  90 #define RTE_NB_STATS (sizeof(rte_stats_strings) / sizeof(rte_stats_strings[0]))
  91
  92 static const struct rte_eth_xstats_name_off rte_rxq_stats_strings[] = {
  93         {"packets", offsetof(struct rte_eth_stats, q_ipackets)},
  94         {"bytes", offsetof(struct rte_eth_stats, q_ibytes)},
  95         {"errors", offsetof(struct rte_eth_stats, q_errors)},
  96 };
  97
  98 #define RTE_NB_RXQ_STATS (sizeof(rte_rxq_stats_strings) /       \
  99                 sizeof(rte_rxq_stats_strings[0]))
 100
 101 static const struct rte_eth_xstats_name_off rte_txq_stats_strings[] = {
 102         {"packets", offsetof(struct rte_eth_stats, q_opackets)},
 103         {"bytes", offsetof(struct rte_eth_stats, q_obytes)},
 104 };
 105 #define RTE_NB_TXQ_STATS (sizeof(rte_txq_stats_strings) /       \
 106                 sizeof(rte_txq_stats_strings[0]))
 107
 108 #define RTE_RX_OFFLOAD_BIT2STR(_name)   \
 109         { DEV_RX_OFFLOAD_##_name, #_name }
 110
 111 static const struct {
 112         uint64_t offload;
 113         const char *name;
 114 } rte_rx_offload_names[] = {
 115         RTE_RX_OFFLOAD_BIT2STR(VLAN_STRIP),
 116         RTE_RX_OFFLOAD_BIT2STR(IPV4_CKSUM),
 117         RTE_RX_OFFLOAD_BIT2STR(UDP_CKSUM),
 118         RTE_RX_OFFLOAD_BIT2STR(TCP_CKSUM),
 119         RTE_RX_OFFLOAD_BIT2STR(TCP_LRO),
 120         RTE_RX_OFFLOAD_BIT2STR(QINQ_STRIP),
 121         RTE_RX_OFFLOAD_BIT2STR(OUTER_IPV4_CKSUM),
 122         RTE_RX_OFFLOAD_BIT2STR(MACSEC_STRIP),
 123         RTE_RX_OFFLOAD_BIT2STR(HEADER_SPLIT),
 124         RTE_RX_OFFLOAD_BIT2STR(VLAN_FILTER),
 125         RTE_RX_OFFLOAD_BIT2STR(VLAN_EXTEND),
 126         RTE_RX_OFFLOAD_BIT2STR(JUMBO_FRAME),
 127         RTE_RX_OFFLOAD_BIT2STR(SCATTER),
 128         RTE_RX_OFFLOAD_BIT2STR(TIMESTAMP),
 129         RTE_RX_OFFLOAD_BIT2STR(SECURITY),
 130         RTE_RX_OFFLOAD_BIT2STR(KEEP_CRC),
 131         RTE_RX_OFFLOAD_BIT2STR(SCTP_CKSUM),
 132         RTE_RX_OFFLOAD_BIT2STR(OUTER_UDP_CKSUM),
 133 };
 134
 135 #undef RTE_RX_OFFLOAD_BIT2STR
 136
 137 #define RTE_TX_OFFLOAD_BIT2STR(_name)   \
 138         { DEV_TX_OFFLOAD_##_name, #_name }
 139
 140 static const struct {
 141         uint64_t offload;
 142         const char *name;
 143 } rte_tx_offload_names[] = {
 144         RTE_TX_OFFLOAD_BIT2STR(VLAN_INSERT),
 145         RTE_TX_OFFLOAD_BIT2STR(IPV4_CKSUM),
 146         RTE_TX_OFFLOAD_BIT2STR(UDP_CKSUM),
 147         RTE_TX_OFFLOAD_BIT2STR(TCP_CKSUM),
 148         RTE_TX_OFFLOAD_BIT2STR(SCTP_CKSUM),
 149         RTE_TX_OFFLOAD_BIT2STR(TCP_TSO),
 150         RTE_TX_OFFLOAD_BIT2STR(UDP_TSO),
 151         RTE_TX_OFFLOAD_BIT2STR(OUTER_IPV4_CKSUM),
 152         RTE_TX_OFFLOAD_BIT2STR(QINQ_INSERT),
 153         RTE_TX_OFFLOAD_BIT2STR(VXLAN_TNL_TSO),
 154         RTE_TX_OFFLOAD_BIT2STR(GRE_TNL_TSO),
 155         RTE_TX_OFFLOAD_BIT2STR(IPIP_TNL_TSO),
 156         RTE_TX_OFFLOAD_BIT2STR(GENEVE_TNL_TSO),
 157         RTE_TX_OFFLOAD_BIT2STR(MACSEC_INSERT),
 158         RTE_TX_OFFLOAD_BIT2STR(MT_LOCKFREE),
 159         RTE_TX_OFFLOAD_BIT2STR(MULTI_SEGS),
 160         RTE_TX_OFFLOAD_BIT2STR(MBUF_FAST_FREE),
 161         RTE_TX_OFFLOAD_BIT2STR(SECURITY),
 162         RTE_TX_OFFLOAD_BIT2STR(UDP_TNL_TSO),
 163         RTE_TX_OFFLOAD_BIT2STR(IP_TNL_TSO),
 164         RTE_TX_OFFLOAD_BIT2STR(OUTER_UDP_CKSUM),
 165         RTE_TX_OFFLOAD_BIT2STR(MATCH_METADATA),
 166 };
 167
 168 #undef RTE_TX_OFFLOAD_BIT2STR
 169
 170 /**
 171  * The user application callback description.
 172  *
 173  * It contains callback address to be registered by user application,
 174  * the pointer to the parameters for callback, and the event type.
 175  */
 176 struct rte_eth_dev_callback {
 177         TAILQ_ENTRY(rte_eth_dev_callback) next; /**< Callbacks list */
 178         rte_eth_dev_cb_fn cb_fn;                /**< Callback address */
 179         void *cb_arg;                           /**< Parameter for callback */
 180         void *ret_param;                        /**< Return parameter */
 181         enum rte_eth_event_type event;          /**< Interrupt event type */
 182         uint32_t active;                        /**< Callback is executing */
 183 };
 184
 185 enum {
 186         STAT_QMAP_TX = 0,
 187         STAT_QMAP_RX
 188 };
 189
 190 int __rte_experimental
 191 rte_eth_iterator_init(struct rte_dev_iterator *iter, const char *devargs_str)
 192 {
 193         int ret;
 194         struct rte_devargs devargs = {.args = NULL};
 195         const char *bus_param_key;
 196         char *bus_str = NULL;
 197         char *cls_str = NULL;
 198         int str_size;
 199
 200         memset(iter, 0, sizeof(*iter));
 201
 202         /*
 203          * The devargs string may use various syntaxes:
 204          *   - 0000:08:00.0,representor=[1-3]
 205          *   - pci:0000:06:00.0,representor=[0,5]
 206          *   - class=eth,mac=00:11:22:33:44:55
 207          * A new syntax is in development (not yet supported):
 208          *   - bus=X,paramX=x/class=Y,paramY=y/driver=Z,paramZ=z
 209          */
 210
 211         /*
 212          * Handle pure class filter (i.e. without any bus-level argument),
 213          * from future new syntax.
 214          * rte_devargs_parse() is not yet supporting the new syntax,
 215          * that's why this simple case is temporarily parsed here.
 216          */
 217 #define iter_anybus_str "class=eth,"
 218         if (strncmp(devargs_str, iter_anybus_str,
 219                         strlen(iter_anybus_str)) == 0) {
 220                 iter->cls_str = devargs_str + strlen(iter_anybus_str);
 221                 goto end;
 222         }
 223
 224         /* Split bus, device and parameters. */
 225         ret = rte_devargs_parse(&devargs, devargs_str);
 226         if (ret != 0)
 227                 goto error;
 228
 229         /*
 230          * Assume parameters of old syntax can match only at ethdev level.
 231          * Extra parameters will be ignored, thanks to "+" prefix.
 232          */
 233         str_size = strlen(devargs.args) + 2;
 234         cls_str = malloc(str_size);
 235         if (cls_str == NULL) {
 236                 ret = -ENOMEM;
 237                 goto error;
 238         }
 239         ret = snprintf(cls_str, str_size, "+%s", devargs.args);
 240         if (ret != str_size - 1) {
 241                 ret = -EINVAL;
 242                 goto error;
 243         }
 244         iter->cls_str = cls_str;
 245         free(devargs.args); /* allocated by rte_devargs_parse() */
 246         devargs.args = NULL;
 247
 248         iter->bus = devargs.bus;
 249         if (iter->bus->dev_iterate == NULL) {
 250                 ret = -ENOTSUP;
 251                 goto error;
 252         }
 253
 254         /* Convert bus args to new syntax for use with new API dev_iterate. */
 255         if (strcmp(iter->bus->name, "vdev") == 0) {
 256                 bus_param_key = "name";
 257         } else if (strcmp(iter->bus->name, "pci") == 0) {
 258                 bus_param_key = "addr";
 259         } else {
 260                 ret = -ENOTSUP;
 261                 goto error;
 262         }
 263         str_size = strlen(bus_param_key) + strlen(devargs.name) + 2;
 264         bus_str = malloc(str_size);
 265         if (bus_str == NULL) {
 266                 ret = -ENOMEM;
 267                 goto error;
 268         }
 269         ret = snprintf(bus_str, str_size, "%s=%s",
 270                         bus_param_key, devargs.name);
 271         if (ret != str_size - 1) {
 272                 ret = -EINVAL;
 273                 goto error;
 274         }
 275         iter->bus_str = bus_str;
 276
 277 end:
 278         iter->cls = rte_class_find_by_name("eth");
 279         return 0;
 280
 281 error:
 282         if (ret == -ENOTSUP)
 283                 RTE_LOG(ERR, EAL, "Bus %s does not support iterating.\n",
 284                                 iter->bus->name);
 285         free(devargs.args);
 286         free(bus_str);
 287         free(cls_str);
 288         return ret;
 289 }
 290
 291 uint16_t __rte_experimental
 292 rte_eth_iterator_next(struct rte_dev_iterator *iter)
 293 {
 294         if (iter->cls == NULL) /* invalid ethdev iterator */
 295                 return RTE_MAX_ETHPORTS;
 296
 297         do { /* loop to try all matching rte_device */
 298                 /* If not pure ethdev filter and */
 299                 if (iter->bus != NULL &&
 300                                 /* not in middle of rte_eth_dev iteration, */
 301                                 iter->class_device == NULL) {
 302                         /* get next rte_device to try. */
 303                         iter->device = iter->bus->dev_iterate(
 304                                         iter->device, iter->bus_str, iter);
 305                         if (iter->device == NULL)
 306                                 break; /* no more rte_device candidate */
 307                 }
 308                 /* A device is matching bus part, need to check ethdev part. */
 309                 iter->class_device = iter->cls->dev_iterate(
 310                                 iter->class_device, iter->cls_str, iter);
 311                 if (iter->class_device != NULL)
 312                         return eth_dev_to_id(iter->class_device); /* match */
 313         } while (iter->bus != NULL); /* need to try next rte_device */
 314
 315         /* No more ethdev port to iterate. */
 316         rte_eth_iterator_cleanup(iter);
 317         return RTE_MAX_ETHPORTS;
 318 }
 319
 320 void __rte_experimental
 321 rte_eth_iterator_cleanup(struct rte_dev_iterator *iter)
 322 {
 323         if (iter->bus_str == NULL)
 324                 return; /* nothing to free in pure class filter */
 325         free(RTE_CAST_FIELD(iter, bus_str, char *)); /* workaround const */
 326         free(RTE_CAST_FIELD(iter, cls_str, char *)); /* workaround const */
 327         memset(iter, 0, sizeof(*iter));
 328 }
 329
 330 uint16_t
 331 rte_eth_find_next(uint16_t port_id)
 332 {
 333         while (port_id < RTE_MAX_ETHPORTS &&
 334                rte_eth_devices[port_id].state != RTE_ETH_DEV_ATTACHED &&
 335                rte_eth_devices[port_id].state != RTE_ETH_DEV_REMOVED)
 336                 port_id++;
 337
 338         if (port_id >= RTE_MAX_ETHPORTS)
 339                 return RTE_MAX_ETHPORTS;
 340
 341         return port_id;
 342 }
 343
 344 static void
 345 rte_eth_dev_shared_data_prepare(void)
 346 {
 347         const unsigned flags = 0;
 348         const struct rte_memzone *mz;
 349
 350         rte_spinlock_lock(&rte_eth_shared_data_lock);
 351
 352         if (rte_eth_dev_shared_data == NULL) {
 353                 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 354                         /* Allocate port data and ownership shared memory. */
 355                         mz = rte_memzone_reserve(MZ_RTE_ETH_DEV_DATA,
 356                                         sizeof(*rte_eth_dev_shared_data),
 357                                         rte_socket_id(), flags);
 358                 } else
 359                         mz = rte_memzone_lookup(MZ_RTE_ETH_DEV_DATA);
 360                 if (mz == NULL)
 361                         rte_panic("Cannot allocate ethdev shared data\n");
 362
 363                 rte_eth_dev_shared_data = mz->addr;
 364                 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 365                         rte_eth_dev_shared_data->next_owner_id =
 366                                         RTE_ETH_DEV_NO_OWNER + 1;
 367                         rte_spinlock_init(&rte_eth_dev_shared_data->ownership_lock);
 368                         memset(rte_eth_dev_shared_data->data, 0,
 369                                sizeof(rte_eth_dev_shared_data->data));
 370                 }
 371         }
 372
 373         rte_spinlock_unlock(&rte_eth_shared_data_lock);
 374 }
 375
 376 static bool
 377 is_allocated(const struct rte_eth_dev *ethdev)
 378 {
 379         return ethdev->data->name[0] != '\0';
 380 }
 381
 382 static struct rte_eth_dev *
 383 _rte_eth_dev_allocated(const char *name)
 384 {
 385         unsigned i;
 386
 387         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 388                 if (rte_eth_devices[i].data != NULL &&
 389                     strcmp(rte_eth_devices[i].data->name, name) == 0)
 390                         return &rte_eth_devices[i];
 391         }
 392         return NULL;
 393 }
 394
 395 struct rte_eth_dev *
 396 rte_eth_dev_allocated(const char *name)
 397 {
 398         struct rte_eth_dev *ethdev;
 399
 400         rte_eth_dev_shared_data_prepare();
 401
 402         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 403
 404         ethdev = _rte_eth_dev_allocated(name);
 405
 406         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 407
 408         return ethdev;
 409 }
 410
 411 static uint16_t
 412 rte_eth_dev_find_free_port(void)
 413 {
 414         unsigned i;
 415
 416         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 417                 /* Using shared name field to find a free port. */
 418                 if (rte_eth_dev_shared_data->data[i].name[0] == '\0') {
 419                         RTE_ASSERT(rte_eth_devices[i].state ==
 420                                    RTE_ETH_DEV_UNUSED);
 421                         return i;
 422                 }
 423         }
 424         return RTE_MAX_ETHPORTS;
 425 }
 426
 427 static struct rte_eth_dev *
 428 eth_dev_get(uint16_t port_id)
 429 {
 430         struct rte_eth_dev *eth_dev = &rte_eth_devices[port_id];
 431
 432         eth_dev->data = &rte_eth_dev_shared_data->data[port_id];
 433
 434         eth_dev_last_created_port = port_id;
 435
 436         return eth_dev;
 437 }
 438
 439 struct rte_eth_dev *
 440 rte_eth_dev_allocate(const char *name)
 441 {
 442         uint16_t port_id;
 443         struct rte_eth_dev *eth_dev = NULL;
 444
 445         rte_eth_dev_shared_data_prepare();
 446
 447         /* Synchronize port creation between primary and secondary threads. */
 448         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 449
 450         if (_rte_eth_dev_allocated(name) != NULL) {
 451                 RTE_ETHDEV_LOG(ERR,
 452                         "Ethernet device with name %s already allocated\n",
 453                         name);
 454                 goto unlock;
 455         }
 456
 457         port_id = rte_eth_dev_find_free_port();
 458         if (port_id == RTE_MAX_ETHPORTS) {
 459                 RTE_ETHDEV_LOG(ERR,
 460                         "Reached maximum number of Ethernet ports\n");
 461                 goto unlock;
 462         }
 463
 464         eth_dev = eth_dev_get(port_id);
 465         snprintf(eth_dev->data->name, sizeof(eth_dev->data->name), "%s", name);
 466         eth_dev->data->port_id = port_id;
 467         eth_dev->data->mtu = ETHER_MTU;
 468
 469 unlock:
 470         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 471
 472         return eth_dev;
 473 }
 474
 475 /*
 476  * Attach to a port already registered by the primary process, which
 477  * makes sure that the same device would have the same port id both
 478  * in the primary and secondary process.
 479  */
 480 struct rte_eth_dev *
 481 rte_eth_dev_attach_secondary(const char *name)
 482 {
 483         uint16_t i;
 484         struct rte_eth_dev *eth_dev = NULL;
 485
 486         rte_eth_dev_shared_data_prepare();
 487
 488         /* Synchronize port attachment to primary port creation and release. */
 489         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 490
 491         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 492                 if (strcmp(rte_eth_dev_shared_data->data[i].name, name) == 0)
 493                         break;
 494         }
 495         if (i == RTE_MAX_ETHPORTS) {
 496                 RTE_ETHDEV_LOG(ERR,
 497                         "Device %s is not driven by the primary process\n",
 498                         name);
 499         } else {
 500                 eth_dev = eth_dev_get(i);
 501                 RTE_ASSERT(eth_dev->data->port_id == i);
 502         }
 503
 504         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 505         return eth_dev;
 506 }
 507
 508 int
 509 rte_eth_dev_release_port(struct rte_eth_dev *eth_dev)
 510 {
 511         if (eth_dev == NULL)
 512                 return -EINVAL;
 513
 514         rte_eth_dev_shared_data_prepare();
 515
 516         if (eth_dev->state != RTE_ETH_DEV_UNUSED)
 517                 _rte_eth_dev_callback_process(eth_dev,
 518                                 RTE_ETH_EVENT_DESTROY, NULL);
 519
 520         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 521
 522         eth_dev->state = RTE_ETH_DEV_UNUSED;
 523
 524         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 525                 rte_free(eth_dev->data->rx_queues);
 526                 rte_free(eth_dev->data->tx_queues);
 527                 rte_free(eth_dev->data->mac_addrs);
 528                 rte_free(eth_dev->data->hash_mac_addrs);
 529                 rte_free(eth_dev->data->dev_private);
 530                 memset(eth_dev->data, 0, sizeof(struct rte_eth_dev_data));
 531         }
 532
 533         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 534
 535         return 0;
 536 }
 537
 538 int
 539 rte_eth_dev_is_valid_port(uint16_t port_id)
 540 {
 541         if (port_id >= RTE_MAX_ETHPORTS ||
 542             (rte_eth_devices[port_id].state == RTE_ETH_DEV_UNUSED))
 543                 return 0;
 544         else
 545                 return 1;
 546 }
 547
 548 static int
 549 rte_eth_is_valid_owner_id(uint64_t owner_id)
 550 {
 551         if (owner_id == RTE_ETH_DEV_NO_OWNER ||
 552             rte_eth_dev_shared_data->next_owner_id <= owner_id)
 553                 return 0;
 554         return 1;
 555 }
 556
 557 uint64_t
 558 rte_eth_find_next_owned_by(uint16_t port_id, const uint64_t owner_id)
 559 {
 560         while (port_id < RTE_MAX_ETHPORTS &&
 561                ((rte_eth_devices[port_id].state != RTE_ETH_DEV_ATTACHED &&
 562                rte_eth_devices[port_id].state != RTE_ETH_DEV_REMOVED) ||
 563                rte_eth_devices[port_id].data->owner.id != owner_id))
 564                 port_id++;
 565
 566         if (port_id >= RTE_MAX_ETHPORTS)
 567                 return RTE_MAX_ETHPORTS;
 568
 569         return port_id;
 570 }
 571
 572 int __rte_experimental
 573 rte_eth_dev_owner_new(uint64_t *owner_id)
 574 {
 575         rte_eth_dev_shared_data_prepare();
 576
 577         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 578
 579         *owner_id = rte_eth_dev_shared_data->next_owner_id++;
 580
 581         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 582         return 0;
 583 }
 584
 585 static int
 586 _rte_eth_dev_owner_set(const uint16_t port_id, const uint64_t old_owner_id,
 587                        const struct rte_eth_dev_owner *new_owner)
 588 {
 589         struct rte_eth_dev *ethdev = &rte_eth_devices[port_id];
 590         struct rte_eth_dev_owner *port_owner;
 591         int sret;
 592
 593         if (port_id >= RTE_MAX_ETHPORTS || !is_allocated(ethdev)) {
 594                 RTE_ETHDEV_LOG(ERR, "Port id %"PRIu16" is not allocated\n",
 595                         port_id);
 596                 return -ENODEV;
 597         }
 598
 599         if (!rte_eth_is_valid_owner_id(new_owner->id) &&
 600             !rte_eth_is_valid_owner_id(old_owner_id)) {
 601                 RTE_ETHDEV_LOG(ERR,
 602                         "Invalid owner old_id=%016"PRIx64" new_id=%016"PRIx64"\n",
 603                        old_owner_id, new_owner->id);
 604                 return -EINVAL;
 605         }
 606
 607         port_owner = &rte_eth_devices[port_id].data->owner;
 608         if (port_owner->id != old_owner_id) {
 609                 RTE_ETHDEV_LOG(ERR,
 610                         "Cannot set owner to port %u already owned by %s_%016"PRIX64"\n",
 611                         port_id, port_owner->name, port_owner->id);
 612                 return -EPERM;
 613         }
 614
 615         sret = snprintf(port_owner->name, RTE_ETH_MAX_OWNER_NAME_LEN, "%s",
 616                         new_owner->name);
 617         if (sret < 0 || sret >= RTE_ETH_MAX_OWNER_NAME_LEN)
 618                 RTE_ETHDEV_LOG(ERR, "Port %u owner name was truncated\n",
 619                         port_id);
 620
 621         port_owner->id = new_owner->id;
 622
 623         RTE_ETHDEV_LOG(DEBUG, "Port %u owner is %s_%016"PRIx64"\n",
 624                 port_id, new_owner->name, new_owner->id);
 625
 626         return 0;
 627 }
 628
 629 int __rte_experimental
 630 rte_eth_dev_owner_set(const uint16_t port_id,
 631                       const struct rte_eth_dev_owner *owner)
 632 {
 633         int ret;
 634
 635         rte_eth_dev_shared_data_prepare();
 636
 637         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 638
 639         ret = _rte_eth_dev_owner_set(port_id, RTE_ETH_DEV_NO_OWNER, owner);
 640
 641         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 642         return ret;
 643 }
 644
 645 int __rte_experimental
 646 rte_eth_dev_owner_unset(const uint16_t port_id, const uint64_t owner_id)
 647 {
 648         const struct rte_eth_dev_owner new_owner = (struct rte_eth_dev_owner)
 649                         {.id = RTE_ETH_DEV_NO_OWNER, .name = ""};
 650         int ret;
 651
 652         rte_eth_dev_shared_data_prepare();
 653
 654         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 655
 656         ret = _rte_eth_dev_owner_set(port_id, owner_id, &new_owner);
 657
 658         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 659         return ret;
 660 }
 661
 662 void __rte_experimental
 663 rte_eth_dev_owner_delete(const uint64_t owner_id)
 664 {
 665         uint16_t port_id;
 666
 667         rte_eth_dev_shared_data_prepare();
 668
 669         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 670
 671         if (rte_eth_is_valid_owner_id(owner_id)) {
 672                 for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++)
 673                         if (rte_eth_devices[port_id].data->owner.id == owner_id)
 674                                 memset(&rte_eth_devices[port_id].data->owner, 0,
 675                                        sizeof(struct rte_eth_dev_owner));
 676                 RTE_ETHDEV_LOG(NOTICE,
 677                         "All port owners owned by %016"PRIx64" identifier have removed\n",
 678                         owner_id);
 679         } else {
 680                 RTE_ETHDEV_LOG(ERR,
 681                                "Invalid owner id=%016"PRIx64"\n",
 682                                owner_id);
 683         }
 684
 685         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 686 }
 687
 688 int __rte_experimental
 689 rte_eth_dev_owner_get(const uint16_t port_id, struct rte_eth_dev_owner *owner)
 690 {
 691         int ret = 0;
 692         struct rte_eth_dev *ethdev = &rte_eth_devices[port_id];
 693
 694         rte_eth_dev_shared_data_prepare();
 695
 696         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 697
 698         if (port_id >= RTE_MAX_ETHPORTS || !is_allocated(ethdev)) {
 699                 RTE_ETHDEV_LOG(ERR, "Port id %"PRIu16" is not allocated\n",
 700                         port_id);
 701                 ret = -ENODEV;
 702         } else {
 703                 rte_memcpy(owner, &ethdev->data->owner, sizeof(*owner));
 704         }
 705
 706         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 707         return ret;
 708 }
 709
 710 int
 711 rte_eth_dev_socket_id(uint16_t port_id)
 712 {
 713         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -1);
 714         return rte_eth_devices[port_id].data->numa_node;
 715 }
 716
 717 void *
 718 rte_eth_dev_get_sec_ctx(uint16_t port_id)
 719 {
 720         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, NULL);
 721         return rte_eth_devices[port_id].security_ctx;
 722 }
 723
 724 uint16_t
 725 rte_eth_dev_count(void)
 726 {
 727         return rte_eth_dev_count_avail();
 728 }
 729
 730 uint16_t
 731 rte_eth_dev_count_avail(void)
 732 {
 733         uint16_t p;
 734         uint16_t count;
 735
 736         count = 0;
 737
 738         RTE_ETH_FOREACH_DEV(p)
 739                 count++;
 740
 741         return count;
 742 }
 743
 744 uint16_t __rte_experimental
 745 rte_eth_dev_count_total(void)
 746 {
 747         uint16_t port, count = 0;
 748
 749         for (port = 0; port < RTE_MAX_ETHPORTS; port++)
 750                 if (rte_eth_devices[port].state != RTE_ETH_DEV_UNUSED)
 751                         count++;
 752
 753         return count;
 754 }
 755
 756 int
 757 rte_eth_dev_get_name_by_port(uint16_t port_id, char *name)
 758 {
 759         char *tmp;
 760
 761         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 762
 763         if (name == NULL) {
 764                 RTE_ETHDEV_LOG(ERR, "Null pointer is specified\n");
 765                 return -EINVAL;
 766         }
 767
 768         /* shouldn't check 'rte_eth_devices[i].data',
 769          * because it might be overwritten by VDEV PMD */
 770         tmp = rte_eth_dev_shared_data->data[port_id].name;
 771         strcpy(name, tmp);
 772         return 0;
 773 }
 774
 775 int
 776 rte_eth_dev_get_port_by_name(const char *name, uint16_t *port_id)
 777 {
 778         uint32_t pid;
 779
 780         if (name == NULL) {
 781                 RTE_ETHDEV_LOG(ERR, "Null pointer is specified\n");
 782                 return -EINVAL;
 783         }
 784
 785         for (pid = 0; pid < RTE_MAX_ETHPORTS; pid++) {
 786                 if (rte_eth_devices[pid].state != RTE_ETH_DEV_UNUSED &&
 787                     !strcmp(name, rte_eth_dev_shared_data->data[pid].name)) {
 788                         *port_id = pid;
 789                         return 0;
 790                 }
 791         }
 792
 793         return -ENODEV;
 794 }
 795
 796 static int
 797 eth_err(uint16_t port_id, int ret)
 798 {
 799         if (ret == 0)
 800                 return 0;
 801         if (rte_eth_dev_is_removed(port_id))
 802                 return -EIO;
 803         return ret;
 804 }
 805
 806 static int
 807 rte_eth_dev_rx_queue_config(struct rte_eth_dev *dev, uint16_t nb_queues)
 808 {
 809         uint16_t old_nb_queues = dev->data->nb_rx_queues;
 810         void **rxq;
 811         unsigned i;
 812
 813         if (dev->data->rx_queues == NULL && nb_queues != 0) { /* first time configuration */
 814                 dev->data->rx_queues = rte_zmalloc("ethdev->rx_queues",
 815                                 sizeof(dev->data->rx_queues[0]) * nb_queues,
 816                                 RTE_CACHE_LINE_SIZE);
 817                 if (dev->data->rx_queues == NULL) {
 818                         dev->data->nb_rx_queues = 0;
 819                         return -(ENOMEM);
 820                 }
 821         } else if (dev->data->rx_queues != NULL && nb_queues != 0) { /* re-configure */
 822                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_release, -ENOTSUP);
 823
 824                 rxq = dev->data->rx_queues;
 825
 826                 for (i = nb_queues; i < old_nb_queues; i++)
 827                         (*dev->dev_ops->rx_queue_release)(rxq[i]);
 828                 rxq = rte_realloc(rxq, sizeof(rxq[0]) * nb_queues,
 829                                 RTE_CACHE_LINE_SIZE);
 830                 if (rxq == NULL)
 831                         return -(ENOMEM);
 832                 if (nb_queues > old_nb_queues) {
 833                         uint16_t new_qs = nb_queues - old_nb_queues;
 834
 835                         memset(rxq + old_nb_queues, 0,
 836                                 sizeof(rxq[0]) * new_qs);
 837                 }
 838
 839                 dev->data->rx_queues = rxq;
 840
 841         } else if (dev->data->rx_queues != NULL && nb_queues == 0) {
 842                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_release, -ENOTSUP);
 843
 844                 rxq = dev->data->rx_queues;
 845
 846                 for (i = nb_queues; i < old_nb_queues; i++)
 847                         (*dev->dev_ops->rx_queue_release)(rxq[i]);
 848
 849                 rte_free(dev->data->rx_queues);
 850                 dev->data->rx_queues = NULL;
 851         }
 852         dev->data->nb_rx_queues = nb_queues;
 853         return 0;
 854 }
 855
 856 int
 857 rte_eth_dev_rx_queue_start(uint16_t port_id, uint16_t rx_queue_id)
 858 {
 859         struct rte_eth_dev *dev;
 860
 861         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 862
 863         dev = &rte_eth_devices[port_id];
 864         if (!dev->data->dev_started) {
 865                 RTE_ETHDEV_LOG(ERR,
 866                         "Port %u must be started before start any queue\n",
 867                         port_id);
 868                 return -EINVAL;
 869         }
 870
 871         if (rx_queue_id >= dev->data->nb_rx_queues) {
 872                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", rx_queue_id);
 873                 return -EINVAL;
 874         }
 875
 876         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_start, -ENOTSUP);
 877
 878         if (dev->data->rx_queue_state[rx_queue_id] != RTE_ETH_QUEUE_STATE_STOPPED) {
 879                 RTE_ETHDEV_LOG(INFO,
 880                         "Queue %"PRIu16" of device with port_id=%"PRIu16" already started\n",
 881                         rx_queue_id, port_id);
 882                 return 0;
 883         }
 884
 885         return eth_err(port_id, dev->dev_ops->rx_queue_start(dev,
 886                                                              rx_queue_id));
 887
 888 }
 889
 890 int
 891 rte_eth_dev_rx_queue_stop(uint16_t port_id, uint16_t rx_queue_id)
 892 {
 893         struct rte_eth_dev *dev;
 894
 895         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 896
 897         dev = &rte_eth_devices[port_id];
 898         if (rx_queue_id >= dev->data->nb_rx_queues) {
 899                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", rx_queue_id);
 900                 return -EINVAL;
 901         }
 902
 903         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_stop, -ENOTSUP);
 904
 905         if (dev->data->rx_queue_state[rx_queue_id] == RTE_ETH_QUEUE_STATE_STOPPED) {
 906                 RTE_ETHDEV_LOG(INFO,
 907                         "Queue %"PRIu16" of device with port_id=%"PRIu16" already stopped\n",
 908                         rx_queue_id, port_id);
 909                 return 0;
 910         }
 911
 912         return eth_err(port_id, dev->dev_ops->rx_queue_stop(dev, rx_queue_id));
 913
 914 }
 915
 916 int
 917 rte_eth_dev_tx_queue_start(uint16_t port_id, uint16_t tx_queue_id)
 918 {
 919         struct rte_eth_dev *dev;
 920
 921         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 922
 923         dev = &rte_eth_devices[port_id];
 924         if (!dev->data->dev_started) {
 925                 RTE_ETHDEV_LOG(ERR,
 926                         "Port %u must be started before start any queue\n",
 927                         port_id);
 928                 return -EINVAL;
 929         }
 930
 931         if (tx_queue_id >= dev->data->nb_tx_queues) {
 932                 RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", tx_queue_id);
 933                 return -EINVAL;
 934         }
 935
 936         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_start, -ENOTSUP);
 937
 938         if (dev->data->tx_queue_state[tx_queue_id] != RTE_ETH_QUEUE_STATE_STOPPED) {
 939                 RTE_ETHDEV_LOG(INFO,
 940                         "Queue %"PRIu16" of device with port_id=%"PRIu16" already started\n",
 941                         tx_queue_id, port_id);
 942                 return 0;
 943         }
 944
 945         return eth_err(port_id, dev->dev_ops->tx_queue_start(dev, tx_queue_id));
 946 }
 947
 948 int
 949 rte_eth_dev_tx_queue_stop(uint16_t port_id, uint16_t tx_queue_id)
 950 {
 951         struct rte_eth_dev *dev;
 952
 953         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 954
 955         dev = &rte_eth_devices[port_id];
 956         if (tx_queue_id >= dev->data->nb_tx_queues) {
 957                 RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", tx_queue_id);
 958                 return -EINVAL;
 959         }
 960
 961         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_stop, -ENOTSUP);
 962
 963         if (dev->data->tx_queue_state[tx_queue_id] == RTE_ETH_QUEUE_STATE_STOPPED) {
 964                 RTE_ETHDEV_LOG(INFO,
 965                         "Queue %"PRIu16" of device with port_id=%"PRIu16" already stopped\n",
 966                         tx_queue_id, port_id);
 967                 return 0;
 968         }
 969
 970         return eth_err(port_id, dev->dev_ops->tx_queue_stop(dev, tx_queue_id));
 971
 972 }
 973
 974 static int
 975 rte_eth_dev_tx_queue_config(struct rte_eth_dev *dev, uint16_t nb_queues)
 976 {
 977         uint16_t old_nb_queues = dev->data->nb_tx_queues;
 978         void **txq;
 979         unsigned i;
 980
 981         if (dev->data->tx_queues == NULL && nb_queues != 0) { /* first time configuration */
 982                 dev->data->tx_queues = rte_zmalloc("ethdev->tx_queues",
 983                                                    sizeof(dev->data->tx_queues[0]) * nb_queues,
 984                                                    RTE_CACHE_LINE_SIZE);
 985                 if (dev->data->tx_queues == NULL) {
 986                         dev->data->nb_tx_queues = 0;
 987                         return -(ENOMEM);
 988                 }
 989         } else if (dev->data->tx_queues != NULL && nb_queues != 0) { /* re-configure */
 990                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_release, -ENOTSUP);
 991
 992                 txq = dev->data->tx_queues;
 993
 994                 for (i = nb_queues; i < old_nb_queues; i++)
 995                         (*dev->dev_ops->tx_queue_release)(txq[i]);
 996                 txq = rte_realloc(txq, sizeof(txq[0]) * nb_queues,
 997                                   RTE_CACHE_LINE_SIZE);
 998                 if (txq == NULL)
 999                         return -ENOMEM;
1000                 if (nb_queues > old_nb_queues) {
1001                         uint16_t new_qs = nb_queues - old_nb_queues;
1002
1003                         memset(txq + old_nb_queues, 0,
1004                                sizeof(txq[0]) * new_qs);
1005                 }
1006
1007                 dev->data->tx_queues = txq;
1008
1009         } else if (dev->data->tx_queues != NULL && nb_queues == 0) {
1010                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_release, -ENOTSUP);
1011
1012                 txq = dev->data->tx_queues;
1013
1014                 for (i = nb_queues; i < old_nb_queues; i++)
1015                         (*dev->dev_ops->tx_queue_release)(txq[i]);
1016
1017                 rte_free(dev->data->tx_queues);
1018                 dev->data->tx_queues = NULL;
1019         }
1020         dev->data->nb_tx_queues = nb_queues;
1021         return 0;
1022 }
1023
1024 uint32_t
1025 rte_eth_speed_bitflag(uint32_t speed, int duplex)
1026 {
1027         switch (speed) {
1028         case ETH_SPEED_NUM_10M:
1029                 return duplex ? ETH_LINK_SPEED_10M : ETH_LINK_SPEED_10M_HD;
1030         case ETH_SPEED_NUM_100M:
1031                 return duplex ? ETH_LINK_SPEED_100M : ETH_LINK_SPEED_100M_HD;
1032         case ETH_SPEED_NUM_1G:
1033                 return ETH_LINK_SPEED_1G;
1034         case ETH_SPEED_NUM_2_5G:
1035                 return ETH_LINK_SPEED_2_5G;
1036         case ETH_SPEED_NUM_5G:
1037                 return ETH_LINK_SPEED_5G;
1038         case ETH_SPEED_NUM_10G:
1039                 return ETH_LINK_SPEED_10G;
1040         case ETH_SPEED_NUM_20G:
1041                 return ETH_LINK_SPEED_20G;
1042         case ETH_SPEED_NUM_25G:
1043                 return ETH_LINK_SPEED_25G;
1044         case ETH_SPEED_NUM_40G:
1045                 return ETH_LINK_SPEED_40G;
1046         case ETH_SPEED_NUM_50G:
1047                 return ETH_LINK_SPEED_50G;
1048         case ETH_SPEED_NUM_56G:
1049                 return ETH_LINK_SPEED_56G;
1050         case ETH_SPEED_NUM_100G:
1051                 return ETH_LINK_SPEED_100G;
1052         default:
1053                 return 0;
1054         }
1055 }
1056
1057 const char *
1058 rte_eth_dev_rx_offload_name(uint64_t offload)
1059 {
1060         const char *name = "UNKNOWN";
1061         unsigned int i;
1062
1063         for (i = 0; i < RTE_DIM(rte_rx_offload_names); ++i) {
1064                 if (offload == rte_rx_offload_names[i].offload) {
1065                         name = rte_rx_offload_names[i].name;
1066                         break;
1067                 }
1068         }
1069
1070         return name;
1071 }
1072
1073 const char *
1074 rte_eth_dev_tx_offload_name(uint64_t offload)
1075 {
1076         const char *name = "UNKNOWN";
1077         unsigned int i;
1078
1079         for (i = 0; i < RTE_DIM(rte_tx_offload_names); ++i) {
1080                 if (offload == rte_tx_offload_names[i].offload) {
1081                         name = rte_tx_offload_names[i].name;
1082                         break;
1083                 }
1084         }
1085
1086         return name;
1087 }
1088
1089 int
1090 rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
1091                       const struct rte_eth_conf *dev_conf)
1092 {
1093         struct rte_eth_dev *dev;
1094         struct rte_eth_dev_info dev_info;
1095         struct rte_eth_conf local_conf = *dev_conf;
1096         int diag;
1097
1098         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1099
1100         dev = &rte_eth_devices[port_id];
1101
1102         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
1103         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_configure, -ENOTSUP);
1104
1105         rte_eth_dev_info_get(port_id, &dev_info);
1106
1107         /* If number of queues specified by application for both Rx and Tx is
1108          * zero, use driver preferred values. This cannot be done individually
1109          * as it is valid for either Tx or Rx (but not both) to be zero.
1110          * If driver does not provide any preferred valued, fall back on
1111          * EAL defaults.
1112          */
1113         if (nb_rx_q == 0 && nb_tx_q == 0) {
1114                 nb_rx_q = dev_info.default_rxportconf.nb_queues;
1115                 if (nb_rx_q == 0)
1116                         nb_rx_q = RTE_ETH_DEV_FALLBACK_RX_NBQUEUES;
1117                 nb_tx_q = dev_info.default_txportconf.nb_queues;
1118                 if (nb_tx_q == 0)
1119                         nb_tx_q = RTE_ETH_DEV_FALLBACK_TX_NBQUEUES;
1120         }
1121
1122         if (nb_rx_q > RTE_MAX_QUEUES_PER_PORT) {
1123                 RTE_ETHDEV_LOG(ERR,
1124                         "Number of RX queues requested (%u) is greater than max supported(%d)\n",
1125                         nb_rx_q, RTE_MAX_QUEUES_PER_PORT);
1126                 return -EINVAL;
1127         }
1128
1129         if (nb_tx_q > RTE_MAX_QUEUES_PER_PORT) {
1130                 RTE_ETHDEV_LOG(ERR,
1131                         "Number of TX queues requested (%u) is greater than max supported(%d)\n",
1132                         nb_tx_q, RTE_MAX_QUEUES_PER_PORT);
1133                 return -EINVAL;
1134         }
1135
1136         if (dev->data->dev_started) {
1137                 RTE_ETHDEV_LOG(ERR,
1138                         "Port %u must be stopped to allow configuration\n",
1139                         port_id);
1140                 return -EBUSY;
1141         }
1142
1143         /* Copy the dev_conf parameter into the dev structure */
1144         memcpy(&dev->data->dev_conf, &local_conf, sizeof(dev->data->dev_conf));
1145
1146         /*
1147          * Check that the numbers of RX and TX queues are not greater
1148          * than the maximum number of RX and TX queues supported by the
1149          * configured device.
1150          */
1151         if (nb_rx_q > dev_info.max_rx_queues) {
1152                 RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%u nb_rx_queues=%u > %u\n",
1153                         port_id, nb_rx_q, dev_info.max_rx_queues);
1154                 return -EINVAL;
1155         }
1156
1157         if (nb_tx_q > dev_info.max_tx_queues) {
1158                 RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%u nb_tx_queues=%u > %u\n",
1159                         port_id, nb_tx_q, dev_info.max_tx_queues);
1160                 return -EINVAL;
1161         }
1162
1163         /* Check that the device supports requested interrupts */
1164         if ((dev_conf->intr_conf.lsc == 1) &&
1165                         (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC))) {
1166                 RTE_ETHDEV_LOG(ERR, "Driver %s does not support lsc\n",
1167                         dev->device->driver->name);
1168                 return -EINVAL;
1169         }
1170         if ((dev_conf->intr_conf.rmv == 1) &&
1171                         (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_RMV))) {
1172                 RTE_ETHDEV_LOG(ERR, "Driver %s does not support rmv\n",
1173                         dev->device->driver->name);
1174                 return -EINVAL;
1175         }
1176
1177         /*
1178          * If jumbo frames are enabled, check that the maximum RX packet
1179          * length is supported by the configured device.
1180          */
1181         if (local_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) {
1182                 if (dev_conf->rxmode.max_rx_pkt_len > dev_info.max_rx_pktlen) {
1183                         RTE_ETHDEV_LOG(ERR,
1184                                 "Ethdev port_id=%u max_rx_pkt_len %u > max valid value %u\n",
1185                                 port_id, dev_conf->rxmode.max_rx_pkt_len,
1186                                 dev_info.max_rx_pktlen);
1187                         return -EINVAL;
1188                 } else if (dev_conf->rxmode.max_rx_pkt_len < ETHER_MIN_LEN) {
1189                         RTE_ETHDEV_LOG(ERR,
1190                                 "Ethdev port_id=%u max_rx_pkt_len %u < min valid value %u\n",
1191                                 port_id, dev_conf->rxmode.max_rx_pkt_len,
1192                                 (unsigned)ETHER_MIN_LEN);
1193                         return -EINVAL;
1194                 }
1195         } else {
1196                 if (dev_conf->rxmode.max_rx_pkt_len < ETHER_MIN_LEN ||
1197                         dev_conf->rxmode.max_rx_pkt_len > ETHER_MAX_LEN)
1198                         /* Use default value */
1199                         dev->data->dev_conf.rxmode.max_rx_pkt_len =
1200                                                         ETHER_MAX_LEN;
1201         }
1202
1203         /* Any requested offloading must be within its device capabilities */
1204         if ((local_conf.rxmode.offloads & dev_info.rx_offload_capa) !=
1205              local_conf.rxmode.offloads) {
1206                 RTE_ETHDEV_LOG(ERR,
1207                         "Ethdev port_id=%u requested Rx offloads 0x%"PRIx64" doesn't match Rx offloads "
1208                         "capabilities 0x%"PRIx64" in %s()\n",
1209                         port_id, local_conf.rxmode.offloads,
1210                         dev_info.rx_offload_capa,
1211                         __func__);
1212                 return -EINVAL;
1213         }
1214         if ((local_conf.txmode.offloads & dev_info.tx_offload_capa) !=
1215              local_conf.txmode.offloads) {
1216                 RTE_ETHDEV_LOG(ERR,
1217                         "Ethdev port_id=%u requested Tx offloads 0x%"PRIx64" doesn't match Tx offloads "
1218                         "capabilities 0x%"PRIx64" in %s()\n",
1219                         port_id, local_conf.txmode.offloads,
1220                         dev_info.tx_offload_capa,
1221                         __func__);
1222                 return -EINVAL;
1223         }
1224
1225         /* Check that device supports requested rss hash functions. */
1226         if ((dev_info.flow_type_rss_offloads |
1227              dev_conf->rx_adv_conf.rss_conf.rss_hf) !=
1228             dev_info.flow_type_rss_offloads) {
1229                 RTE_ETHDEV_LOG(ERR,
1230                         "Ethdev port_id=%u invalid rss_hf: 0x%"PRIx64", valid value: 0x%"PRIx64"\n",
1231                         port_id, dev_conf->rx_adv_conf.rss_conf.rss_hf,
1232                         dev_info.flow_type_rss_offloads);
1233                 return -EINVAL;
1234         }
1235
1236         /*
1237          * Setup new number of RX/TX queues and reconfigure device.
1238          */
1239         diag = rte_eth_dev_rx_queue_config(dev, nb_rx_q);
1240         if (diag != 0) {
1241                 RTE_ETHDEV_LOG(ERR,
1242                         "Port%u rte_eth_dev_rx_queue_config = %d\n",
1243                         port_id, diag);
1244                 return diag;
1245         }
1246
1247         diag = rte_eth_dev_tx_queue_config(dev, nb_tx_q);
1248         if (diag != 0) {
1249                 RTE_ETHDEV_LOG(ERR,
1250                         "Port%u rte_eth_dev_tx_queue_config = %d\n",
1251                         port_id, diag);
1252                 rte_eth_dev_rx_queue_config(dev, 0);
1253                 return diag;
1254         }
1255
1256         diag = (*dev->dev_ops->dev_configure)(dev);
1257         if (diag != 0) {
1258                 RTE_ETHDEV_LOG(ERR, "Port%u dev_configure = %d\n",
1259                         port_id, diag);
1260                 rte_eth_dev_rx_queue_config(dev, 0);
1261                 rte_eth_dev_tx_queue_config(dev, 0);
1262                 return eth_err(port_id, diag);
1263         }
1264
1265         /* Initialize Rx profiling if enabled at compilation time. */
1266         diag = __rte_eth_dev_profile_init(port_id, dev);
1267         if (diag != 0) {
1268                 RTE_ETHDEV_LOG(ERR, "Port%u __rte_eth_dev_profile_init = %d\n",
1269                         port_id, diag);
1270                 rte_eth_dev_rx_queue_config(dev, 0);
1271                 rte_eth_dev_tx_queue_config(dev, 0);
1272                 return eth_err(port_id, diag);
1273         }
1274
1275         return 0;
1276 }
1277
1278 void
1279 _rte_eth_dev_reset(struct rte_eth_dev *dev)
1280 {
1281         if (dev->data->dev_started) {
1282                 RTE_ETHDEV_LOG(ERR, "Port %u must be stopped to allow reset\n",
1283                         dev->data->port_id);
1284                 return;
1285         }
1286
1287         rte_eth_dev_rx_queue_config(dev, 0);
1288         rte_eth_dev_tx_queue_config(dev, 0);
1289
1290         memset(&dev->data->dev_conf, 0, sizeof(dev->data->dev_conf));
1291 }
1292
1293 static void
1294 rte_eth_dev_mac_restore(struct rte_eth_dev *dev,
1295                         struct rte_eth_dev_info *dev_info)
1296 {
1297         struct ether_addr *addr;
1298         uint16_t i;
1299         uint32_t pool = 0;
1300         uint64_t pool_mask;
1301
1302         /* replay MAC address configuration including default MAC */
1303         addr = &dev->data->mac_addrs[0];
1304         if (*dev->dev_ops->mac_addr_set != NULL)
1305                 (*dev->dev_ops->mac_addr_set)(dev, addr);
1306         else if (*dev->dev_ops->mac_addr_add != NULL)
1307                 (*dev->dev_ops->mac_addr_add)(dev, addr, 0, pool);
1308
1309         if (*dev->dev_ops->mac_addr_add != NULL) {
1310                 for (i = 1; i < dev_info->max_mac_addrs; i++) {
1311                         addr = &dev->data->mac_addrs[i];
1312
1313                         /* skip zero address */
1314                         if (is_zero_ether_addr(addr))
1315                                 continue;
1316
1317                         pool = 0;
1318                         pool_mask = dev->data->mac_pool_sel[i];
1319
1320                         do {
1321                                 if (pool_mask & 1ULL)
1322                                         (*dev->dev_ops->mac_addr_add)(dev,
1323                                                 addr, i, pool);
1324                                 pool_mask >>= 1;
1325                                 pool++;
1326                         } while (pool_mask);
1327                 }
1328         }
1329 }
1330
1331 static void
1332 rte_eth_dev_config_restore(struct rte_eth_dev *dev,
1333                            struct rte_eth_dev_info *dev_info, uint16_t port_id)
1334 {
1335         if (!(*dev_info->dev_flags & RTE_ETH_DEV_NOLIVE_MAC_ADDR))
1336                 rte_eth_dev_mac_restore(dev, dev_info);
1337
1338         /* replay promiscuous configuration */
1339         if (rte_eth_promiscuous_get(port_id) == 1)
1340                 rte_eth_promiscuous_enable(port_id);
1341         else if (rte_eth_promiscuous_get(port_id) == 0)
1342                 rte_eth_promiscuous_disable(port_id);
1343
1344         /* replay all multicast configuration */
1345         if (rte_eth_allmulticast_get(port_id) == 1)
1346                 rte_eth_allmulticast_enable(port_id);
1347         else if (rte_eth_allmulticast_get(port_id) == 0)
1348                 rte_eth_allmulticast_disable(port_id);
1349 }
1350
1351 int
1352 rte_eth_dev_start(uint16_t port_id)
1353 {
1354         struct rte_eth_dev *dev;
1355         struct rte_eth_dev_info dev_info;
1356         int diag;
1357
1358         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1359
1360         dev = &rte_eth_devices[port_id];
1361
1362         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_start, -ENOTSUP);
1363
1364         if (dev->data->dev_started != 0) {
1365                 RTE_ETHDEV_LOG(INFO,
1366                         "Device with port_id=%"PRIu16" already started\n",
1367                         port_id);
1368                 return 0;
1369         }
1370
1371         rte_eth_dev_info_get(port_id, &dev_info);
1372
1373         /* Lets restore MAC now if device does not support live change */
1374         if (*dev_info.dev_flags & RTE_ETH_DEV_NOLIVE_MAC_ADDR)
1375                 rte_eth_dev_mac_restore(dev, &dev_info);
1376
1377         diag = (*dev->dev_ops->dev_start)(dev);
1378         if (diag == 0)
1379                 dev->data->dev_started = 1;
1380         else
1381                 return eth_err(port_id, diag);
1382
1383         rte_eth_dev_config_restore(dev, &dev_info, port_id);
1384
1385         if (dev->data->dev_conf.intr_conf.lsc == 0) {
1386                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->link_update, -ENOTSUP);
1387                 (*dev->dev_ops->link_update)(dev, 0);
1388         }
1389         return 0;
1390 }
1391
1392 void
1393 rte_eth_dev_stop(uint16_t port_id)
1394 {
1395         struct rte_eth_dev *dev;
1396
1397         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1398         dev = &rte_eth_devices[port_id];
1399
1400         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_stop);
1401
1402         if (dev->data->dev_started == 0) {
1403                 RTE_ETHDEV_LOG(INFO,
1404                         "Device with port_id=%"PRIu16" already stopped\n",
1405                         port_id);
1406                 return;
1407         }
1408
1409         dev->data->dev_started = 0;
1410         (*dev->dev_ops->dev_stop)(dev);
1411 }
1412
1413 int
1414 rte_eth_dev_set_link_up(uint16_t port_id)
1415 {
1416         struct rte_eth_dev *dev;
1417
1418         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1419
1420         dev = &rte_eth_devices[port_id];
1421
1422         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_set_link_up, -ENOTSUP);
1423         return eth_err(port_id, (*dev->dev_ops->dev_set_link_up)(dev));
1424 }
1425
1426 int
1427 rte_eth_dev_set_link_down(uint16_t port_id)
1428 {
1429         struct rte_eth_dev *dev;
1430
1431         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1432
1433         dev = &rte_eth_devices[port_id];
1434
1435         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_set_link_down, -ENOTSUP);
1436         return eth_err(port_id, (*dev->dev_ops->dev_set_link_down)(dev));
1437 }
1438
1439 void
1440 rte_eth_dev_close(uint16_t port_id)
1441 {
1442         struct rte_eth_dev *dev;
1443
1444         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1445         dev = &rte_eth_devices[port_id];
1446
1447         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_close);
1448         dev->data->dev_started = 0;
1449         (*dev->dev_ops->dev_close)(dev);
1450
1451         /* check behaviour flag - temporary for PMD migration */
1452         if ((dev->data->dev_flags & RTE_ETH_DEV_CLOSE_REMOVE) != 0) {
1453                 /* new behaviour: send event + reset state + free all data */
1454                 rte_eth_dev_release_port(dev);
1455                 return;
1456         }
1457         RTE_ETHDEV_LOG(DEBUG, "Port closing is using an old behaviour.\n"
1458                         "The driver %s should migrate to the new behaviour.\n",
1459                         dev->device->driver->name);
1460         /* old behaviour: only free queue arrays */
1461         dev->data->nb_rx_queues = 0;
1462         rte_free(dev->data->rx_queues);
1463         dev->data->rx_queues = NULL;
1464         dev->data->nb_tx_queues = 0;
1465         rte_free(dev->data->tx_queues);
1466         dev->data->tx_queues = NULL;
1467 }
1468
1469 int
1470 rte_eth_dev_reset(uint16_t port_id)
1471 {
1472         struct rte_eth_dev *dev;
1473         int ret;
1474
1475         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1476         dev = &rte_eth_devices[port_id];
1477
1478         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_reset, -ENOTSUP);
1479
1480         rte_eth_dev_stop(port_id);
1481         ret = dev->dev_ops->dev_reset(dev);
1482
1483         return eth_err(port_id, ret);
1484 }
1485
1486 int __rte_experimental
1487 rte_eth_dev_is_removed(uint16_t port_id)
1488 {
1489         struct rte_eth_dev *dev;
1490         int ret;
1491
1492         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, 0);
1493
1494         dev = &rte_eth_devices[port_id];
1495
1496         if (dev->state == RTE_ETH_DEV_REMOVED)
1497                 return 1;
1498
1499         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->is_removed, 0);
1500
1501         ret = dev->dev_ops->is_removed(dev);
1502         if (ret != 0)
1503                 /* Device is physically removed. */
1504                 dev->state = RTE_ETH_DEV_REMOVED;
1505
1506         return ret;
1507 }
1508
1509 int
1510 rte_eth_rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
1511                        uint16_t nb_rx_desc, unsigned int socket_id,
1512                        const struct rte_eth_rxconf *rx_conf,
1513                        struct rte_mempool *mp)
1514 {
1515         int ret;
1516         uint32_t mbp_buf_size;
1517         struct rte_eth_dev *dev;
1518         struct rte_eth_dev_info dev_info;
1519         struct rte_eth_rxconf local_conf;
1520         void **rxq;
1521
1522         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1523
1524         dev = &rte_eth_devices[port_id];
1525         if (rx_queue_id >= dev->data->nb_rx_queues) {
1526                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", rx_queue_id);
1527                 return -EINVAL;
1528         }
1529
1530         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
1531         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_setup, -ENOTSUP);
1532
1533         /*
1534          * Check the size of the mbuf data buffer.
1535          * This value must be provided in the private data of the memory pool.
1536          * First check that the memory pool has a valid private data.
1537          */
1538         rte_eth_dev_info_get(port_id, &dev_info);
1539         if (mp->private_data_size < sizeof(struct rte_pktmbuf_pool_private)) {
1540                 RTE_ETHDEV_LOG(ERR, "%s private_data_size %d < %d\n",
1541                         mp->name, (int)mp->private_data_size,
1542                         (int)sizeof(struct rte_pktmbuf_pool_private));
1543                 return -ENOSPC;
1544         }
1545         mbp_buf_size = rte_pktmbuf_data_room_size(mp);
1546
1547         if ((mbp_buf_size - RTE_PKTMBUF_HEADROOM) < dev_info.min_rx_bufsize) {
1548                 RTE_ETHDEV_LOG(ERR,
1549                         "%s mbuf_data_room_size %d < %d (RTE_PKTMBUF_HEADROOM=%d + min_rx_bufsize(dev)=%d)\n",
1550                         mp->name, (int)mbp_buf_size,
1551                         (int)(RTE_PKTMBUF_HEADROOM + dev_info.min_rx_bufsize),
1552                         (int)RTE_PKTMBUF_HEADROOM,
1553                         (int)dev_info.min_rx_bufsize);
1554                 return -EINVAL;
1555         }
1556
1557         /* Use default specified by driver, if nb_rx_desc is zero */
1558         if (nb_rx_desc == 0) {
1559                 nb_rx_desc = dev_info.default_rxportconf.ring_size;
1560                 /* If driver default is also zero, fall back on EAL default */
1561                 if (nb_rx_desc == 0)
1562                         nb_rx_desc = RTE_ETH_DEV_FALLBACK_RX_RINGSIZE;
1563         }
1564
1565         if (nb_rx_desc > dev_info.rx_desc_lim.nb_max ||
1566                         nb_rx_desc < dev_info.rx_desc_lim.nb_min ||
1567                         nb_rx_desc % dev_info.rx_desc_lim.nb_align != 0) {
1568
1569                 RTE_ETHDEV_LOG(ERR,
1570                         "Invalid value for nb_rx_desc(=%hu), should be: <= %hu, = %hu, and a product of %hu\n",
1571                         nb_rx_desc, dev_info.rx_desc_lim.nb_max,
1572                         dev_info.rx_desc_lim.nb_min,
1573                         dev_info.rx_desc_lim.nb_align);
1574                 return -EINVAL;
1575         }
1576
1577         if (dev->data->dev_started &&
1578                 !(dev_info.dev_capa &
1579                         RTE_ETH_DEV_CAPA_RUNTIME_RX_QUEUE_SETUP))
1580                 return -EBUSY;
1581
1582         if (dev->data->dev_started &&
1583                 (dev->data->rx_queue_state[rx_queue_id] !=
1584                         RTE_ETH_QUEUE_STATE_STOPPED))
1585                 return -EBUSY;
1586
1587         rxq = dev->data->rx_queues;
1588         if (rxq[rx_queue_id]) {
1589                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_release,
1590                                         -ENOTSUP);
1591                 (*dev->dev_ops->rx_queue_release)(rxq[rx_queue_id]);
1592                 rxq[rx_queue_id] = NULL;
1593         }
1594
1595         if (rx_conf == NULL)
1596                 rx_conf = &dev_info.default_rxconf;
1597
1598         local_conf = *rx_conf;
1599
1600         /*
1601          * If an offloading has already been enabled in
1602          * rte_eth_dev_configure(), it has been enabled on all queues,
1603          * so there is no need to enable it in this queue again.
1604          * The local_conf.offloads input to underlying PMD only carries
1605          * those offloadings which are only enabled on this queue and
1606          * not enabled on all queues.
1607          */
1608         local_conf.offloads &= ~dev->data->dev_conf.rxmode.offloads;
1609
1610         /*
1611          * New added offloadings for this queue are those not enabled in
1612          * rte_eth_dev_configure() and they must be per-queue type.
1613          * A pure per-port offloading can't be enabled on a queue while
1614          * disabled on another queue. A pure per-port offloading can't
1615          * be enabled for any queue as new added one if it hasn't been
1616          * enabled in rte_eth_dev_configure().
1617          */
1618         if ((local_conf.offloads & dev_info.rx_queue_offload_capa) !=
1619              local_conf.offloads) {
1620                 RTE_ETHDEV_LOG(ERR,
1621                         "Ethdev port_id=%d rx_queue_id=%d, new added offloads 0x%"PRIx64" must be "
1622                         "within pre-queue offload capabilities 0x%"PRIx64" in %s()\n",
1623                         port_id, rx_queue_id, local_conf.offloads,
1624                         dev_info.rx_queue_offload_capa,
1625                         __func__);
1626                 return -EINVAL;
1627         }
1628
1629         ret = (*dev->dev_ops->rx_queue_setup)(dev, rx_queue_id, nb_rx_desc,
1630                                               socket_id, &local_conf, mp);
1631         if (!ret) {
1632                 if (!dev->data->min_rx_buf_size ||
1633                     dev->data->min_rx_buf_size > mbp_buf_size)
1634                         dev->data->min_rx_buf_size = mbp_buf_size;
1635         }
1636
1637         return eth_err(port_id, ret);
1638 }
1639
1640 int
1641 rte_eth_tx_queue_setup(uint16_t port_id, uint16_t tx_queue_id,
1642                        uint16_t nb_tx_desc, unsigned int socket_id,
1643                        const struct rte_eth_txconf *tx_conf)
1644 {
1645         struct rte_eth_dev *dev;
1646         struct rte_eth_dev_info dev_info;
1647         struct rte_eth_txconf local_conf;
1648         void **txq;
1649
1650         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1651
1652         dev = &rte_eth_devices[port_id];
1653         if (tx_queue_id >= dev->data->nb_tx_queues) {
1654                 RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", tx_queue_id);
1655                 return -EINVAL;
1656         }
1657
1658         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
1659         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_setup, -ENOTSUP);
1660
1661         rte_eth_dev_info_get(port_id, &dev_info);
1662
1663         /* Use default specified by driver, if nb_tx_desc is zero */
1664         if (nb_tx_desc == 0) {
1665                 nb_tx_desc = dev_info.default_txportconf.ring_size;
1666                 /* If driver default is zero, fall back on EAL default */
1667                 if (nb_tx_desc == 0)
1668                         nb_tx_desc = RTE_ETH_DEV_FALLBACK_TX_RINGSIZE;
1669         }
1670         if (nb_tx_desc > dev_info.tx_desc_lim.nb_max ||
1671             nb_tx_desc < dev_info.tx_desc_lim.nb_min ||
1672             nb_tx_desc % dev_info.tx_desc_lim.nb_align != 0) {
1673                 RTE_ETHDEV_LOG(ERR,
1674                         "Invalid value for nb_tx_desc(=%hu), should be: <= %hu, = %hu, and a product of %hu\n",
1675                         nb_tx_desc, dev_info.tx_desc_lim.nb_max,
1676                         dev_info.tx_desc_lim.nb_min,
1677                         dev_info.tx_desc_lim.nb_align);
1678                 return -EINVAL;
1679         }
1680
1681         if (dev->data->dev_started &&
1682                 !(dev_info.dev_capa &
1683                         RTE_ETH_DEV_CAPA_RUNTIME_TX_QUEUE_SETUP))
1684                 return -EBUSY;
1685
1686         if (dev->data->dev_started &&
1687                 (dev->data->tx_queue_state[tx_queue_id] !=
1688                         RTE_ETH_QUEUE_STATE_STOPPED))
1689                 return -EBUSY;
1690
1691         txq = dev->data->tx_queues;
1692         if (txq[tx_queue_id]) {
1693                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_release,
1694                                         -ENOTSUP);
1695                 (*dev->dev_ops->tx_queue_release)(txq[tx_queue_id]);
1696                 txq[tx_queue_id] = NULL;
1697         }
1698
1699         if (tx_conf == NULL)
1700                 tx_conf = &dev_info.default_txconf;
1701
1702         local_conf = *tx_conf;
1703
1704         /*
1705          * If an offloading has already been enabled in
1706          * rte_eth_dev_configure(), it has been enabled on all queues,
1707          * so there is no need to enable it in this queue again.
1708          * The local_conf.offloads input to underlying PMD only carries
1709          * those offloadings which are only enabled on this queue and
1710          * not enabled on all queues.
1711          */
1712         local_conf.offloads &= ~dev->data->dev_conf.txmode.offloads;
1713
1714         /*
1715          * New added offloadings for this queue are those not enabled in
1716          * rte_eth_dev_configure() and they must be per-queue type.
1717          * A pure per-port offloading can't be enabled on a queue while
1718          * disabled on another queue. A pure per-port offloading can't
1719          * be enabled for any queue as new added one if it hasn't been
1720          * enabled in rte_eth_dev_configure().
1721          */
1722         if ((local_conf.offloads & dev_info.tx_queue_offload_capa) !=
1723              local_conf.offloads) {
1724                 RTE_ETHDEV_LOG(ERR,
1725                         "Ethdev port_id=%d tx_queue_id=%d, new added offloads 0x%"PRIx64" must be "
1726                         "within pre-queue offload capabilities 0x%"PRIx64" in %s()\n",
1727                         port_id, tx_queue_id, local_conf.offloads,
1728                         dev_info.tx_queue_offload_capa,
1729                         __func__);
1730                 return -EINVAL;
1731         }
1732
1733         return eth_err(port_id, (*dev->dev_ops->tx_queue_setup)(dev,
1734                        tx_queue_id, nb_tx_desc, socket_id, &local_conf));
1735 }
1736
1737 void
1738 rte_eth_tx_buffer_drop_callback(struct rte_mbuf **pkts, uint16_t unsent,
1739                 void *userdata __rte_unused)
1740 {
1741         unsigned i;
1742
1743         for (i = 0; i < unsent; i++)
1744                 rte_pktmbuf_free(pkts[i]);
1745 }
1746
1747 void
1748 rte_eth_tx_buffer_count_callback(struct rte_mbuf **pkts, uint16_t unsent,
1749                 void *userdata)
1750 {
1751         uint64_t *count = userdata;
1752         unsigned i;
1753
1754         for (i = 0; i < unsent; i++)
1755                 rte_pktmbuf_free(pkts[i]);
1756
1757         *count += unsent;
1758 }
1759
1760 int
1761 rte_eth_tx_buffer_set_err_callback(struct rte_eth_dev_tx_buffer *buffer,
1762                 buffer_tx_error_fn cbfn, void *userdata)
1763 {
1764         buffer->error_callback = cbfn;
1765         buffer->error_userdata = userdata;
1766         return 0;
1767 }
1768
1769 int
1770 rte_eth_tx_buffer_init(struct rte_eth_dev_tx_buffer *buffer, uint16_t size)
1771 {
1772         int ret = 0;
1773
1774         if (buffer == NULL)
1775                 return -EINVAL;
1776
1777         buffer->size = size;
1778         if (buffer->error_callback == NULL) {
1779                 ret = rte_eth_tx_buffer_set_err_callback(
1780                         buffer, rte_eth_tx_buffer_drop_callback, NULL);
1781         }
1782
1783         return ret;
1784 }
1785
1786 int
1787 rte_eth_tx_done_cleanup(uint16_t port_id, uint16_t queue_id, uint32_t free_cnt)
1788 {
1789         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
1790         int ret;
1791
1792         /* Validate Input Data. Bail if not valid or not supported. */
1793         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1794         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_done_cleanup, -ENOTSUP);
1795
1796         /* Call driver to free pending mbufs. */
1797         ret = (*dev->dev_ops->tx_done_cleanup)(dev->data->tx_queues[queue_id],
1798                                                free_cnt);
1799         return eth_err(port_id, ret);
1800 }
1801
1802 void
1803 rte_eth_promiscuous_enable(uint16_t port_id)
1804 {
1805         struct rte_eth_dev *dev;
1806
1807         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1808         dev = &rte_eth_devices[port_id];
1809
1810         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->promiscuous_enable);
1811         (*dev->dev_ops->promiscuous_enable)(dev);
1812         dev->data->promiscuous = 1;
1813 }
1814
1815 void
1816 rte_eth_promiscuous_disable(uint16_t port_id)
1817 {
1818         struct rte_eth_dev *dev;
1819
1820         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1821         dev = &rte_eth_devices[port_id];
1822
1823         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->promiscuous_disable);
1824         dev->data->promiscuous = 0;
1825         (*dev->dev_ops->promiscuous_disable)(dev);
1826 }
1827
1828 int
1829 rte_eth_promiscuous_get(uint16_t port_id)
1830 {
1831         struct rte_eth_dev *dev;
1832
1833         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1834
1835         dev = &rte_eth_devices[port_id];
1836         return dev->data->promiscuous;
1837 }
1838
1839 void
1840 rte_eth_allmulticast_enable(uint16_t port_id)
1841 {
1842         struct rte_eth_dev *dev;
1843
1844         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1845         dev = &rte_eth_devices[port_id];
1846
1847         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->allmulticast_enable);
1848         (*dev->dev_ops->allmulticast_enable)(dev);
1849         dev->data->all_multicast = 1;
1850 }
1851
1852 void
1853 rte_eth_allmulticast_disable(uint16_t port_id)
1854 {
1855         struct rte_eth_dev *dev;
1856
1857         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1858         dev = &rte_eth_devices[port_id];
1859
1860         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->allmulticast_disable);
1861         dev->data->all_multicast = 0;
1862         (*dev->dev_ops->allmulticast_disable)(dev);
1863 }
1864
1865 int
1866 rte_eth_allmulticast_get(uint16_t port_id)
1867 {
1868         struct rte_eth_dev *dev;
1869
1870         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1871
1872         dev = &rte_eth_devices[port_id];
1873         return dev->data->all_multicast;
1874 }
1875
1876 void
1877 rte_eth_link_get(uint16_t port_id, struct rte_eth_link *eth_link)
1878 {
1879         struct rte_eth_dev *dev;
1880
1881         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1882         dev = &rte_eth_devices[port_id];
1883
1884         if (dev->data->dev_conf.intr_conf.lsc &&
1885             dev->data->dev_started)
1886                 rte_eth_linkstatus_get(dev, eth_link);
1887         else {
1888                 RTE_FUNC_PTR_OR_RET(*dev->dev_ops->link_update);
1889                 (*dev->dev_ops->link_update)(dev, 1);
1890                 *eth_link = dev->data->dev_link;
1891         }
1892 }
1893
1894 void
1895 rte_eth_link_get_nowait(uint16_t port_id, struct rte_eth_link *eth_link)
1896 {
1897         struct rte_eth_dev *dev;
1898
1899         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1900         dev = &rte_eth_devices[port_id];
1901
1902         if (dev->data->dev_conf.intr_conf.lsc &&
1903             dev->data->dev_started)
1904                 rte_eth_linkstatus_get(dev, eth_link);
1905         else {
1906                 RTE_FUNC_PTR_OR_RET(*dev->dev_ops->link_update);
1907                 (*dev->dev_ops->link_update)(dev, 0);
1908                 *eth_link = dev->data->dev_link;
1909         }
1910 }
1911
1912 int
1913 rte_eth_stats_get(uint16_t port_id, struct rte_eth_stats *stats)
1914 {
1915         struct rte_eth_dev *dev;
1916
1917         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1918
1919         dev = &rte_eth_devices[port_id];
1920         memset(stats, 0, sizeof(*stats));
1921
1922         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_get, -ENOTSUP);
1923         stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed;
1924         return eth_err(port_id, (*dev->dev_ops->stats_get)(dev, stats));
1925 }
1926
1927 int
1928 rte_eth_stats_reset(uint16_t port_id)
1929 {
1930         struct rte_eth_dev *dev;
1931
1932         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1933         dev = &rte_eth_devices[port_id];
1934
1935         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_reset, -ENOTSUP);
1936         (*dev->dev_ops->stats_reset)(dev);
1937         dev->data->rx_mbuf_alloc_failed = 0;
1938
1939         return 0;
1940 }
1941
1942 static inline int
1943 get_xstats_basic_count(struct rte_eth_dev *dev)
1944 {
1945         uint16_t nb_rxqs, nb_txqs;
1946         int count;
1947
1948         nb_rxqs = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
1949         nb_txqs = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
1950
1951         count = RTE_NB_STATS;
1952         count += nb_rxqs * RTE_NB_RXQ_STATS;
1953         count += nb_txqs * RTE_NB_TXQ_STATS;
1954
1955         return count;
1956 }
1957
1958 static int
1959 get_xstats_count(uint16_t port_id)
1960 {
1961         struct rte_eth_dev *dev;
1962         int count;
1963
1964         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1965         dev = &rte_eth_devices[port_id];
1966         if (dev->dev_ops->xstats_get_names_by_id != NULL) {
1967                 count = (*dev->dev_ops->xstats_get_names_by_id)(dev, NULL,
1968                                 NULL, 0);
1969                 if (count < 0)
1970                         return eth_err(port_id, count);
1971         }
1972         if (dev->dev_ops->xstats_get_names != NULL) {
1973                 count = (*dev->dev_ops->xstats_get_names)(dev, NULL, 0);
1974                 if (count < 0)
1975                         return eth_err(port_id, count);
1976         } else
1977                 count = 0;
1978
1979
1980         count += get_xstats_basic_count(dev);
1981
1982         return count;
1983 }
1984
1985 int
1986 rte_eth_xstats_get_id_by_name(uint16_t port_id, const char *xstat_name,
1987                 uint64_t *id)
1988 {
1989         int cnt_xstats, idx_xstat;
1990
1991         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1992
1993         if (!id) {
1994                 RTE_ETHDEV_LOG(ERR, "Id pointer is NULL\n");
1995                 return -ENOMEM;
1996         }
1997
1998         if (!xstat_name) {
1999                 RTE_ETHDEV_LOG(ERR, "xstat_name pointer is NULL\n");
2000                 return -ENOMEM;
2001         }
2002
2003         /* Get count */
2004         cnt_xstats = rte_eth_xstats_get_names_by_id(port_id, NULL, 0, NULL);
2005         if (cnt_xstats  < 0) {
2006                 RTE_ETHDEV_LOG(ERR, "Cannot get count of xstats\n");
2007                 return -ENODEV;
2008         }
2009
2010         /* Get id-name lookup table */
2011         struct rte_eth_xstat_name xstats_names[cnt_xstats];
2012
2013         if (cnt_xstats != rte_eth_xstats_get_names_by_id(
2014                         port_id, xstats_names, cnt_xstats, NULL)) {
2015                 RTE_ETHDEV_LOG(ERR, "Cannot get xstats lookup\n");
2016                 return -1;
2017         }
2018
2019         for (idx_xstat = 0; idx_xstat < cnt_xstats; idx_xstat++) {
2020                 if (!strcmp(xstats_names[idx_xstat].name, xstat_name)) {
2021                         *id = idx_xstat;
2022                         return 0;
2023                 };
2024         }
2025
2026         return -EINVAL;
2027 }
2028
2029 /* retrieve basic stats names */
2030 static int
2031 rte_eth_basic_stats_get_names(struct rte_eth_dev *dev,
2032         struct rte_eth_xstat_name *xstats_names)
2033 {
2034         int cnt_used_entries = 0;
2035         uint32_t idx, id_queue;
2036         uint16_t num_q;
2037
2038         for (idx = 0; idx < RTE_NB_STATS; idx++) {
2039                 snprintf(xstats_names[cnt_used_entries].name,
2040                         sizeof(xstats_names[0].name),
2041                         "%s", rte_stats_strings[idx].name);
2042                 cnt_used_entries++;
2043         }
2044         num_q = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2045         for (id_queue = 0; id_queue < num_q; id_queue++) {
2046                 for (idx = 0; idx < RTE_NB_RXQ_STATS; idx++) {
2047                         snprintf(xstats_names[cnt_used_entries].name,
2048                                 sizeof(xstats_names[0].name),
2049                                 "rx_q%u%s",
2050                                 id_queue, rte_rxq_stats_strings[idx].name);
2051                         cnt_used_entries++;
2052                 }
2053
2054         }
2055         num_q = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2056         for (id_queue = 0; id_queue < num_q; id_queue++) {
2057                 for (idx = 0; idx < RTE_NB_TXQ_STATS; idx++) {
2058                         snprintf(xstats_names[cnt_used_entries].name,
2059                                 sizeof(xstats_names[0].name),
2060                                 "tx_q%u%s",
2061                                 id_queue, rte_txq_stats_strings[idx].name);
2062                         cnt_used_entries++;
2063                 }
2064         }
2065         return cnt_used_entries;
2066 }
2067
2068 /* retrieve ethdev extended statistics names */
2069 int
2070 rte_eth_xstats_get_names_by_id(uint16_t port_id,
2071         struct rte_eth_xstat_name *xstats_names, unsigned int size,
2072         uint64_t *ids)
2073 {
2074         struct rte_eth_xstat_name *xstats_names_copy;
2075         unsigned int no_basic_stat_requested = 1;
2076         unsigned int no_ext_stat_requested = 1;
2077         unsigned int expected_entries;
2078         unsigned int basic_count;
2079         struct rte_eth_dev *dev;
2080         unsigned int i;
2081         int ret;
2082
2083         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2084         dev = &rte_eth_devices[port_id];
2085
2086         basic_count = get_xstats_basic_count(dev);
2087         ret = get_xstats_count(port_id);
2088         if (ret < 0)
2089                 return ret;
2090         expected_entries = (unsigned int)ret;
2091
2092         /* Return max number of stats if no ids given */
2093         if (!ids) {
2094                 if (!xstats_names)
2095                         return expected_entries;
2096                 else if (xstats_names && size < expected_entries)
2097                         return expected_entries;
2098         }
2099
2100         if (ids && !xstats_names)
2101                 return -EINVAL;
2102
2103         if (ids && dev->dev_ops->xstats_get_names_by_id != NULL && size > 0) {
2104                 uint64_t ids_copy[size];
2105
2106                 for (i = 0; i < size; i++) {
2107                         if (ids[i] < basic_count) {
2108                                 no_basic_stat_requested = 0;
2109                                 break;
2110                         }
2111
2112                         /*
2113                          * Convert ids to xstats ids that PMD knows.
2114                          * ids known by user are basic + extended stats.
2115                          */
2116                         ids_copy[i] = ids[i] - basic_count;
2117                 }
2118
2119                 if (no_basic_stat_requested)
2120                         return (*dev->dev_ops->xstats_get_names_by_id)(dev,
2121                                         xstats_names, ids_copy, size);
2122         }
2123
2124         /* Retrieve all stats */
2125         if (!ids) {
2126                 int num_stats = rte_eth_xstats_get_names(port_id, xstats_names,
2127                                 expected_entries);
2128                 if (num_stats < 0 || num_stats > (int)expected_entries)
2129                         return num_stats;
2130                 else
2131                         return expected_entries;
2132         }
2133
2134         xstats_names_copy = calloc(expected_entries,
2135                 sizeof(struct rte_eth_xstat_name));
2136
2137         if (!xstats_names_copy) {
2138                 RTE_ETHDEV_LOG(ERR, "Can't allocate memory\n");
2139                 return -ENOMEM;
2140         }
2141
2142         if (ids) {
2143                 for (i = 0; i < size; i++) {
2144                         if (ids[i] >= basic_count) {
2145                                 no_ext_stat_requested = 0;
2146                                 break;
2147                         }
2148                 }
2149         }
2150
2151         /* Fill xstats_names_copy structure */
2152         if (ids && no_ext_stat_requested) {
2153                 rte_eth_basic_stats_get_names(dev, xstats_names_copy);
2154         } else {
2155                 ret = rte_eth_xstats_get_names(port_id, xstats_names_copy,
2156                         expected_entries);
2157                 if (ret < 0) {
2158                         free(xstats_names_copy);
2159                         return ret;
2160                 }
2161         }
2162
2163         /* Filter stats */
2164         for (i = 0; i < size; i++) {
2165                 if (ids[i] >= expected_entries) {
2166                         RTE_ETHDEV_LOG(ERR, "Id value isn't valid\n");
2167                         free(xstats_names_copy);
2168                         return -1;
2169                 }
2170                 xstats_names[i] = xstats_names_copy[ids[i]];
2171         }
2172
2173         free(xstats_names_copy);
2174         return size;
2175 }
2176
2177 int
2178 rte_eth_xstats_get_names(uint16_t port_id,
2179         struct rte_eth_xstat_name *xstats_names,
2180         unsigned int size)
2181 {
2182         struct rte_eth_dev *dev;
2183         int cnt_used_entries;
2184         int cnt_expected_entries;
2185         int cnt_driver_entries;
2186
2187         cnt_expected_entries = get_xstats_count(port_id);
2188         if (xstats_names == NULL || cnt_expected_entries < 0 ||
2189                         (int)size < cnt_expected_entries)
2190                 return cnt_expected_entries;
2191
2192         /* port_id checked in get_xstats_count() */
2193         dev = &rte_eth_devices[port_id];
2194
2195         cnt_used_entries = rte_eth_basic_stats_get_names(
2196                 dev, xstats_names);
2197
2198         if (dev->dev_ops->xstats_get_names != NULL) {
2199                 /* If there are any driver-specific xstats, append them
2200                  * to end of list.
2201                  */
2202                 cnt_driver_entries = (*dev->dev_ops->xstats_get_names)(
2203                         dev,
2204                         xstats_names + cnt_used_entries,
2205                         size - cnt_used_entries);
2206                 if (cnt_driver_entries < 0)
2207                         return eth_err(port_id, cnt_driver_entries);
2208                 cnt_used_entries += cnt_driver_entries;
2209         }
2210
2211         return cnt_used_entries;
2212 }
2213
2214
2215 static int
2216 rte_eth_basic_stats_get(uint16_t port_id, struct rte_eth_xstat *xstats)
2217 {
2218         struct rte_eth_dev *dev;
2219         struct rte_eth_stats eth_stats;
2220         unsigned int count = 0, i, q;
2221         uint64_t val, *stats_ptr;
2222         uint16_t nb_rxqs, nb_txqs;
2223         int ret;
2224
2225         ret = rte_eth_stats_get(port_id, &eth_stats);
2226         if (ret < 0)
2227                 return ret;
2228
2229         dev = &rte_eth_devices[port_id];
2230
2231         nb_rxqs = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2232         nb_txqs = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2233
2234         /* global stats */
2235         for (i = 0; i < RTE_NB_STATS; i++) {
2236                 stats_ptr = RTE_PTR_ADD(&eth_stats,
2237                                         rte_stats_strings[i].offset);
2238                 val = *stats_ptr;
2239                 xstats[count++].value = val;
2240         }
2241
2242         /* per-rxq stats */
2243         for (q = 0; q < nb_rxqs; q++) {
2244                 for (i = 0; i < RTE_NB_RXQ_STATS; i++) {
2245                         stats_ptr = RTE_PTR_ADD(&eth_stats,
2246                                         rte_rxq_stats_strings[i].offset +
2247                                         q * sizeof(uint64_t));
2248                         val = *stats_ptr;
2249                         xstats[count++].value = val;
2250                 }
2251         }
2252
2253         /* per-txq stats */
2254         for (q = 0; q < nb_txqs; q++) {
2255                 for (i = 0; i < RTE_NB_TXQ_STATS; i++) {
2256                         stats_ptr = RTE_PTR_ADD(&eth_stats,
2257                                         rte_txq_stats_strings[i].offset +
2258                                         q * sizeof(uint64_t));
2259                         val = *stats_ptr;
2260                         xstats[count++].value = val;
2261                 }
2262         }
2263         return count;
2264 }
2265
2266 /* retrieve ethdev extended statistics */
2267 int
2268 rte_eth_xstats_get_by_id(uint16_t port_id, const uint64_t *ids,
2269                          uint64_t *values, unsigned int size)
2270 {
2271         unsigned int no_basic_stat_requested = 1;
2272         unsigned int no_ext_stat_requested = 1;
2273         unsigned int num_xstats_filled;
2274         unsigned int basic_count;
2275         uint16_t expected_entries;
2276         struct rte_eth_dev *dev;
2277         unsigned int i;
2278         int ret;
2279
2280         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2281         ret = get_xstats_count(port_id);
2282         if (ret < 0)
2283                 return ret;
2284         expected_entries = (uint16_t)ret;
2285         struct rte_eth_xstat xstats[expected_entries];
2286         dev = &rte_eth_devices[port_id];
2287         basic_count = get_xstats_basic_count(dev);
2288
2289         /* Return max number of stats if no ids given */
2290         if (!ids) {
2291                 if (!values)
2292                         return expected_entries;
2293                 else if (values && size < expected_entries)
2294                         return expected_entries;
2295         }
2296
2297         if (ids && !values)
2298                 return -EINVAL;
2299
2300         if (ids && dev->dev_ops->xstats_get_by_id != NULL && size) {
2301                 unsigned int basic_count = get_xstats_basic_count(dev);
2302                 uint64_t ids_copy[size];
2303
2304                 for (i = 0; i < size; i++) {
2305                         if (ids[i] < basic_count) {
2306                                 no_basic_stat_requested = 0;
2307                                 break;
2308                         }
2309
2310                         /*
2311                          * Convert ids to xstats ids that PMD knows.
2312                          * ids known by user are basic + extended stats.
2313                          */
2314                         ids_copy[i] = ids[i] - basic_count;
2315                 }
2316
2317                 if (no_basic_stat_requested)
2318                         return (*dev->dev_ops->xstats_get_by_id)(dev, ids_copy,
2319                                         values, size);
2320         }
2321
2322         if (ids) {
2323                 for (i = 0; i < size; i++) {
2324                         if (ids[i] >= basic_count) {
2325                                 no_ext_stat_requested = 0;
2326                                 break;
2327                         }
2328                 }
2329         }
2330
2331         /* Fill the xstats structure */
2332         if (ids && no_ext_stat_requested)
2333                 ret = rte_eth_basic_stats_get(port_id, xstats);
2334         else
2335                 ret = rte_eth_xstats_get(port_id, xstats, expected_entries);
2336
2337         if (ret < 0)
2338                 return ret;
2339         num_xstats_filled = (unsigned int)ret;
2340
2341         /* Return all stats */
2342         if (!ids) {
2343                 for (i = 0; i < num_xstats_filled; i++)
2344                         values[i] = xstats[i].value;
2345                 return expected_entries;
2346         }
2347
2348         /* Filter stats */
2349         for (i = 0; i < size; i++) {
2350                 if (ids[i] >= expected_entries) {
2351                         RTE_ETHDEV_LOG(ERR, "Id value isn't valid\n");
2352                         return -1;
2353                 }
2354                 values[i] = xstats[ids[i]].value;
2355         }
2356         return size;
2357 }
2358
2359 int
2360 rte_eth_xstats_get(uint16_t port_id, struct rte_eth_xstat *xstats,
2361         unsigned int n)
2362 {
2363         struct rte_eth_dev *dev;
2364         unsigned int count = 0, i;
2365         signed int xcount = 0;
2366         uint16_t nb_rxqs, nb_txqs;
2367         int ret;
2368
2369         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
2370
2371         dev = &rte_eth_devices[port_id];
2372
2373         nb_rxqs = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2374         nb_txqs = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2375
2376         /* Return generic statistics */
2377         count = RTE_NB_STATS + (nb_rxqs * RTE_NB_RXQ_STATS) +
2378                 (nb_txqs * RTE_NB_TXQ_STATS);
2379
2380         /* implemented by the driver */
2381         if (dev->dev_ops->xstats_get != NULL) {
2382                 /* Retrieve the xstats from the driver at the end of the
2383                  * xstats struct.
2384                  */
2385                 xcount = (*dev->dev_ops->xstats_get)(dev,
2386                                      xstats ? xstats + count : NULL,
2387                                      (n > count) ? n - count : 0);
2388
2389                 if (xcount < 0)
2390                         return eth_err(port_id, xcount);
2391         }
2392
2393         if (n < count + xcount || xstats == NULL)
2394                 return count + xcount;
2395
2396         /* now fill the xstats structure */
2397         ret = rte_eth_basic_stats_get(port_id, xstats);
2398         if (ret < 0)
2399                 return ret;
2400         count = ret;
2401
2402         for (i = 0; i < count; i++)
2403                 xstats[i].id = i;
2404         /* add an offset to driver-specific stats */
2405         for ( ; i < count + xcount; i++)
2406                 xstats[i].id += count;
2407
2408         return count + xcount;
2409 }
2410
2411 /* reset ethdev extended statistics */
2412 void
2413 rte_eth_xstats_reset(uint16_t port_id)
2414 {
2415         struct rte_eth_dev *dev;
2416
2417         RTE_ETH_VALID_PORTID_OR_RET(port_id);
2418         dev = &rte_eth_devices[port_id];
2419
2420         /* implemented by the driver */
2421         if (dev->dev_ops->xstats_reset != NULL) {
2422                 (*dev->dev_ops->xstats_reset)(dev);
2423                 return;
2424         }
2425
2426         /* fallback to default */
2427         rte_eth_stats_reset(port_id);
2428 }
2429
2430 static int
2431 set_queue_stats_mapping(uint16_t port_id, uint16_t queue_id, uint8_t stat_idx,
2432                 uint8_t is_rx)
2433 {
2434         struct rte_eth_dev *dev;
2435
2436         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2437
2438         dev = &rte_eth_devices[port_id];
2439
2440         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_stats_mapping_set, -ENOTSUP);
2441
2442         if (is_rx && (queue_id >= dev->data->nb_rx_queues))
2443                 return -EINVAL;
2444
2445         if (!is_rx && (queue_id >= dev->data->nb_tx_queues))
2446                 return -EINVAL;
2447
2448         if (stat_idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)
2449                 return -EINVAL;
2450
2451         return (*dev->dev_ops->queue_stats_mapping_set)
2452                         (dev, queue_id, stat_idx, is_rx);
2453 }
2454
2455
2456 int
2457 rte_eth_dev_set_tx_queue_stats_mapping(uint16_t port_id, uint16_t tx_queue_id,
2458                 uint8_t stat_idx)
2459 {
2460         return eth_err(port_id, set_queue_stats_mapping(port_id, tx_queue_id,
2461                                                 stat_idx, STAT_QMAP_TX));
2462 }
2463
2464
2465 int
2466 rte_eth_dev_set_rx_queue_stats_mapping(uint16_t port_id, uint16_t rx_queue_id,
2467                 uint8_t stat_idx)
2468 {
2469         return eth_err(port_id, set_queue_stats_mapping(port_id, rx_queue_id,
2470                                                 stat_idx, STAT_QMAP_RX));
2471 }
2472
2473 int
2474 rte_eth_dev_fw_version_get(uint16_t port_id, char *fw_version, size_t fw_size)
2475 {
2476         struct rte_eth_dev *dev;
2477
2478         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2479         dev = &rte_eth_devices[port_id];
2480
2481         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->fw_version_get, -ENOTSUP);
2482         return eth_err(port_id, (*dev->dev_ops->fw_version_get)(dev,
2483                                                         fw_version, fw_size));
2484 }
2485
2486 void
2487 rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info *dev_info)
2488 {
2489         struct rte_eth_dev *dev;
2490         const struct rte_eth_desc_lim lim = {
2491                 .nb_max = UINT16_MAX,
2492                 .nb_min = 0,
2493                 .nb_align = 1,
2494         };
2495
2496         RTE_ETH_VALID_PORTID_OR_RET(port_id);
2497         dev = &rte_eth_devices[port_id];
2498
2499         memset(dev_info, 0, sizeof(struct rte_eth_dev_info));
2500         dev_info->rx_desc_lim = lim;
2501         dev_info->tx_desc_lim = lim;
2502         dev_info->device = dev->device;
2503
2504         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_infos_get);
2505         (*dev->dev_ops->dev_infos_get)(dev, dev_info);
2506         dev_info->driver_name = dev->device->driver->name;
2507         dev_info->nb_rx_queues = dev->data->nb_rx_queues;
2508         dev_info->nb_tx_queues = dev->data->nb_tx_queues;
2509
2510         dev_info->dev_flags = &dev->data->dev_flags;
2511 }
2512
2513 int
2514 rte_eth_dev_get_supported_ptypes(uint16_t port_id, uint32_t ptype_mask,
2515                                  uint32_t *ptypes, int num)
2516 {
2517         int i, j;
2518         struct rte_eth_dev *dev;
2519         const uint32_t *all_ptypes;
2520
2521         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2522         dev = &rte_eth_devices[port_id];
2523         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_supported_ptypes_get, 0);
2524         all_ptypes = (*dev->dev_ops->dev_supported_ptypes_get)(dev);
2525
2526         if (!all_ptypes)
2527                 return 0;
2528
2529         for (i = 0, j = 0; all_ptypes[i] != RTE_PTYPE_UNKNOWN; ++i)
2530                 if (all_ptypes[i] & ptype_mask) {
2531                         if (j < num)
2532                                 ptypes[j] = all_ptypes[i];
2533                         j++;
2534                 }
2535
2536         return j;
2537 }
2538
2539 void
2540 rte_eth_macaddr_get(uint16_t port_id, struct ether_addr *mac_addr)
2541 {
2542         struct rte_eth_dev *dev;
2543
2544         RTE_ETH_VALID_PORTID_OR_RET(port_id);
2545         dev = &rte_eth_devices[port_id];
2546         ether_addr_copy(&dev->data->mac_addrs[0], mac_addr);
2547 }
2548
2549
2550 int
2551 rte_eth_dev_get_mtu(uint16_t port_id, uint16_t *mtu)
2552 {
2553         struct rte_eth_dev *dev;
2554
2555         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2556
2557         dev = &rte_eth_devices[port_id];
2558         *mtu = dev->data->mtu;
2559         return 0;
2560 }
2561
2562 int
2563 rte_eth_dev_set_mtu(uint16_t port_id, uint16_t mtu)
2564 {
2565         int ret;
2566         struct rte_eth_dev *dev;
2567
2568         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2569         dev = &rte_eth_devices[port_id];
2570         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mtu_set, -ENOTSUP);
2571
2572         ret = (*dev->dev_ops->mtu_set)(dev, mtu);
2573         if (!ret)
2574                 dev->data->mtu = mtu;
2575
2576         return eth_err(port_id, ret);
2577 }
2578
2579 int
2580 rte_eth_dev_vlan_filter(uint16_t port_id, uint16_t vlan_id, int on)
2581 {
2582         struct rte_eth_dev *dev;
2583         int ret;
2584
2585         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2586         dev = &rte_eth_devices[port_id];
2587         if (!(dev->data->dev_conf.rxmode.offloads &
2588               DEV_RX_OFFLOAD_VLAN_FILTER)) {
2589                 RTE_ETHDEV_LOG(ERR, "Port %u: vlan-filtering disabled\n",
2590                         port_id);
2591                 return -ENOSYS;
2592         }
2593
2594         if (vlan_id > 4095) {
2595                 RTE_ETHDEV_LOG(ERR, "Port_id=%u invalid vlan_id=%u > 4095\n",
2596                         port_id, vlan_id);
2597                 return -EINVAL;
2598         }
2599         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_filter_set, -ENOTSUP);
2600
2601         ret = (*dev->dev_ops->vlan_filter_set)(dev, vlan_id, on);
2602         if (ret == 0) {
2603                 struct rte_vlan_filter_conf *vfc;
2604                 int vidx;
2605                 int vbit;
2606
2607                 vfc = &dev->data->vlan_filter_conf;
2608                 vidx = vlan_id / 64;
2609                 vbit = vlan_id % 64;
2610
2611                 if (on)
2612                         vfc->ids[vidx] |= UINT64_C(1) << vbit;
2613                 else
2614                         vfc->ids[vidx] &= ~(UINT64_C(1) << vbit);
2615         }
2616
2617         return eth_err(port_id, ret);
2618 }
2619
2620 int
2621 rte_eth_dev_set_vlan_strip_on_queue(uint16_t port_id, uint16_t rx_queue_id,
2622                                     int on)
2623 {
2624         struct rte_eth_dev *dev;
2625
2626         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2627         dev = &rte_eth_devices[port_id];
2628         if (rx_queue_id >= dev->data->nb_rx_queues) {
2629                 RTE_ETHDEV_LOG(ERR, "Invalid rx_queue_id=%u\n", rx_queue_id);
2630                 return -EINVAL;
2631         }
2632
2633         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_strip_queue_set, -ENOTSUP);
2634         (*dev->dev_ops->vlan_strip_queue_set)(dev, rx_queue_id, on);
2635
2636         return 0;
2637 }
2638
2639 int
2640 rte_eth_dev_set_vlan_ether_type(uint16_t port_id,
2641                                 enum rte_vlan_type vlan_type,
2642                                 uint16_t tpid)
2643 {
2644         struct rte_eth_dev *dev;
2645
2646         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2647         dev = &rte_eth_devices[port_id];
2648         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_tpid_set, -ENOTSUP);
2649
2650         return eth_err(port_id, (*dev->dev_ops->vlan_tpid_set)(dev, vlan_type,
2651                                                                tpid));
2652 }
2653
2654 int
2655 rte_eth_dev_set_vlan_offload(uint16_t port_id, int offload_mask)
2656 {
2657         struct rte_eth_dev *dev;
2658         int ret = 0;
2659         int mask = 0;
2660         int cur, org = 0;
2661         uint64_t orig_offloads;
2662
2663         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2664         dev = &rte_eth_devices[port_id];
2665
2666         /* save original values in case of failure */
2667         orig_offloads = dev->data->dev_conf.rxmode.offloads;
2668
2669         /*check which option changed by application*/
2670         cur = !!(offload_mask & ETH_VLAN_STRIP_OFFLOAD);
2671         org = !!(dev->data->dev_conf.rxmode.offloads &
2672                  DEV_RX_OFFLOAD_VLAN_STRIP);
2673         if (cur != org) {
2674                 if (cur)
2675                         dev->data->dev_conf.rxmode.offloads |=
2676                                 DEV_RX_OFFLOAD_VLAN_STRIP;
2677                 else
2678                         dev->data->dev_conf.rxmode.offloads &=
2679                                 ~DEV_RX_OFFLOAD_VLAN_STRIP;
2680                 mask |= ETH_VLAN_STRIP_MASK;
2681         }
2682
2683         cur = !!(offload_mask & ETH_VLAN_FILTER_OFFLOAD);
2684         org = !!(dev->data->dev_conf.rxmode.offloads &
2685                  DEV_RX_OFFLOAD_VLAN_FILTER);
2686         if (cur != org) {
2687                 if (cur)
2688                         dev->data->dev_conf.rxmode.offloads |=
2689                                 DEV_RX_OFFLOAD_VLAN_FILTER;
2690                 else
2691                         dev->data->dev_conf.rxmode.offloads &=
2692                                 ~DEV_RX_OFFLOAD_VLAN_FILTER;
2693                 mask |= ETH_VLAN_FILTER_MASK;
2694         }
2695
2696         cur = !!(offload_mask & ETH_VLAN_EXTEND_OFFLOAD);
2697         org = !!(dev->data->dev_conf.rxmode.offloads &
2698                  DEV_RX_OFFLOAD_VLAN_EXTEND);
2699         if (cur != org) {
2700                 if (cur)
2701                         dev->data->dev_conf.rxmode.offloads |=
2702                                 DEV_RX_OFFLOAD_VLAN_EXTEND;
2703                 else
2704                         dev->data->dev_conf.rxmode.offloads &=
2705                                 ~DEV_RX_OFFLOAD_VLAN_EXTEND;
2706                 mask |= ETH_VLAN_EXTEND_MASK;
2707         }
2708
2709         /*no change*/
2710         if (mask == 0)
2711                 return ret;
2712
2713         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_offload_set, -ENOTSUP);
2714         ret = (*dev->dev_ops->vlan_offload_set)(dev, mask);
2715         if (ret) {
2716                 /* hit an error restore  original values */
2717                 dev->data->dev_conf.rxmode.offloads = orig_offloads;
2718         }
2719
2720         return eth_err(port_id, ret);
2721 }
2722
2723 int
2724 rte_eth_dev_get_vlan_offload(uint16_t port_id)
2725 {
2726         struct rte_eth_dev *dev;
2727         int ret = 0;
2728
2729         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2730         dev = &rte_eth_devices[port_id];
2731
2732         if (dev->data->dev_conf.rxmode.offloads &
2733             DEV_RX_OFFLOAD_VLAN_STRIP)
2734                 ret |= ETH_VLAN_STRIP_OFFLOAD;
2735
2736         if (dev->data->dev_conf.rxmode.offloads &
2737             DEV_RX_OFFLOAD_VLAN_FILTER)
2738                 ret |= ETH_VLAN_FILTER_OFFLOAD;
2739
2740         if (dev->data->dev_conf.rxmode.offloads &
2741             DEV_RX_OFFLOAD_VLAN_EXTEND)
2742                 ret |= ETH_VLAN_EXTEND_OFFLOAD;
2743
2744         return ret;
2745 }
2746
2747 int
2748 rte_eth_dev_set_vlan_pvid(uint16_t port_id, uint16_t pvid, int on)
2749 {
2750         struct rte_eth_dev *dev;
2751
2752         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2753         dev = &rte_eth_devices[port_id];
2754         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_pvid_set, -ENOTSUP);
2755
2756         return eth_err(port_id, (*dev->dev_ops->vlan_pvid_set)(dev, pvid, on));
2757 }
2758
2759 int
2760 rte_eth_dev_flow_ctrl_get(uint16_t port_id, struct rte_eth_fc_conf *fc_conf)
2761 {
2762         struct rte_eth_dev *dev;
2763
2764         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2765         dev = &rte_eth_devices[port_id];
2766         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->flow_ctrl_get, -ENOTSUP);
2767         memset(fc_conf, 0, sizeof(*fc_conf));
2768         return eth_err(port_id, (*dev->dev_ops->flow_ctrl_get)(dev, fc_conf));
2769 }
2770
2771 int
2772 rte_eth_dev_flow_ctrl_set(uint16_t port_id, struct rte_eth_fc_conf *fc_conf)
2773 {
2774         struct rte_eth_dev *dev;
2775
2776         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2777         if ((fc_conf->send_xon != 0) && (fc_conf->send_xon != 1)) {
2778                 RTE_ETHDEV_LOG(ERR, "Invalid send_xon, only 0/1 allowed\n");
2779                 return -EINVAL;
2780         }
2781
2782         dev = &rte_eth_devices[port_id];
2783         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->flow_ctrl_set, -ENOTSUP);
2784         return eth_err(port_id, (*dev->dev_ops->flow_ctrl_set)(dev, fc_conf));
2785 }
2786
2787 int
2788 rte_eth_dev_priority_flow_ctrl_set(uint16_t port_id,
2789                                    struct rte_eth_pfc_conf *pfc_conf)
2790 {
2791         struct rte_eth_dev *dev;
2792
2793         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2794         if (pfc_conf->priority > (ETH_DCB_NUM_USER_PRIORITIES - 1)) {
2795                 RTE_ETHDEV_LOG(ERR, "Invalid priority, only 0-7 allowed\n");
2796                 return -EINVAL;
2797         }
2798
2799         dev = &rte_eth_devices[port_id];
2800         /* High water, low water validation are device specific */
2801         if  (*dev->dev_ops->priority_flow_ctrl_set)
2802                 return eth_err(port_id, (*dev->dev_ops->priority_flow_ctrl_set)
2803                                         (dev, pfc_conf));
2804         return -ENOTSUP;
2805 }
2806
2807 static int
2808 rte_eth_check_reta_mask(struct rte_eth_rss_reta_entry64 *reta_conf,
2809                         uint16_t reta_size)
2810 {
2811         uint16_t i, num;
2812
2813         if (!reta_conf)
2814                 return -EINVAL;
2815
2816         num = (reta_size + RTE_RETA_GROUP_SIZE - 1) / RTE_RETA_GROUP_SIZE;
2817         for (i = 0; i < num; i++) {
2818                 if (reta_conf[i].mask)
2819                         return 0;
2820         }
2821
2822         return -EINVAL;
2823 }
2824
2825 static int
2826 rte_eth_check_reta_entry(struct rte_eth_rss_reta_entry64 *reta_conf,
2827                          uint16_t reta_size,
2828                          uint16_t max_rxq)
2829 {
2830         uint16_t i, idx, shift;
2831
2832         if (!reta_conf)
2833                 return -EINVAL;
2834
2835         if (max_rxq == 0) {
2836                 RTE_ETHDEV_LOG(ERR, "No receive queue is available\n");
2837                 return -EINVAL;
2838         }
2839
2840         for (i = 0; i < reta_size; i++) {
2841                 idx = i / RTE_RETA_GROUP_SIZE;
2842                 shift = i % RTE_RETA_GROUP_SIZE;
2843                 if ((reta_conf[idx].mask & (1ULL << shift)) &&
2844                         (reta_conf[idx].reta[shift] >= max_rxq)) {
2845                         RTE_ETHDEV_LOG(ERR,
2846                                 "reta_conf[%u]->reta[%u]: %u exceeds the maximum rxq index: %u\n",
2847                                 idx, shift,
2848                                 reta_conf[idx].reta[shift], max_rxq);
2849                         return -EINVAL;
2850                 }
2851         }
2852
2853         return 0;
2854 }
2855
2856 int
2857 rte_eth_dev_rss_reta_update(uint16_t port_id,
2858                             struct rte_eth_rss_reta_entry64 *reta_conf,
2859                             uint16_t reta_size)
2860 {
2861         struct rte_eth_dev *dev;
2862         int ret;
2863
2864         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2865         /* Check mask bits */
2866         ret = rte_eth_check_reta_mask(reta_conf, reta_size);
2867         if (ret < 0)
2868                 return ret;
2869
2870         dev = &rte_eth_devices[port_id];
2871
2872         /* Check entry value */
2873         ret = rte_eth_check_reta_entry(reta_conf, reta_size,
2874                                 dev->data->nb_rx_queues);
2875         if (ret < 0)
2876                 return ret;
2877
2878         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->reta_update, -ENOTSUP);
2879         return eth_err(port_id, (*dev->dev_ops->reta_update)(dev, reta_conf,
2880                                                              reta_size));
2881 }
2882
2883 int
2884 rte_eth_dev_rss_reta_query(uint16_t port_id,
2885                            struct rte_eth_rss_reta_entry64 *reta_conf,
2886                            uint16_t reta_size)
2887 {
2888         struct rte_eth_dev *dev;
2889         int ret;
2890
2891         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2892
2893         /* Check mask bits */
2894         ret = rte_eth_check_reta_mask(reta_conf, reta_size);
2895         if (ret < 0)
2896                 return ret;
2897
2898         dev = &rte_eth_devices[port_id];
2899         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->reta_query, -ENOTSUP);
2900         return eth_err(port_id, (*dev->dev_ops->reta_query)(dev, reta_conf,
2901                                                             reta_size));
2902 }
2903
2904 int
2905 rte_eth_dev_rss_hash_update(uint16_t port_id,
2906                             struct rte_eth_rss_conf *rss_conf)
2907 {
2908         struct rte_eth_dev *dev;
2909         struct rte_eth_dev_info dev_info = { .flow_type_rss_offloads = 0, };
2910
2911         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2912         dev = &rte_eth_devices[port_id];
2913         rte_eth_dev_info_get(port_id, &dev_info);
2914         if ((dev_info.flow_type_rss_offloads | rss_conf->rss_hf) !=
2915             dev_info.flow_type_rss_offloads) {
2916                 RTE_ETHDEV_LOG(ERR,
2917                         "Ethdev port_id=%u invalid rss_hf: 0x%"PRIx64", valid value: 0x%"PRIx64"\n",
2918                         port_id, rss_conf->rss_hf,
2919                         dev_info.flow_type_rss_offloads);
2920                 return -EINVAL;
2921         }
2922         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rss_hash_update, -ENOTSUP);
2923         return eth_err(port_id, (*dev->dev_ops->rss_hash_update)(dev,
2924                                                                  rss_conf));
2925 }
2926
2927 int
2928 rte_eth_dev_rss_hash_conf_get(uint16_t port_id,
2929                               struct rte_eth_rss_conf *rss_conf)
2930 {
2931         struct rte_eth_dev *dev;
2932
2933         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2934         dev = &rte_eth_devices[port_id];
2935         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rss_hash_conf_get, -ENOTSUP);
2936         return eth_err(port_id, (*dev->dev_ops->rss_hash_conf_get)(dev,
2937                                                                    rss_conf));
2938 }
2939
2940 int
2941 rte_eth_dev_udp_tunnel_port_add(uint16_t port_id,
2942                                 struct rte_eth_udp_tunnel *udp_tunnel)
2943 {
2944         struct rte_eth_dev *dev;
2945
2946         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2947         if (udp_tunnel == NULL) {
2948                 RTE_ETHDEV_LOG(ERR, "Invalid udp_tunnel parameter\n");
2949                 return -EINVAL;
2950         }
2951
2952         if (udp_tunnel->prot_type >= RTE_TUNNEL_TYPE_MAX) {
2953                 RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
2954                 return -EINVAL;
2955         }
2956
2957         dev = &rte_eth_devices[port_id];
2958         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->udp_tunnel_port_add, -ENOTSUP);
2959         return eth_err(port_id, (*dev->dev_ops->udp_tunnel_port_add)(dev,
2960                                                                 udp_tunnel));
2961 }
2962
2963 int
2964 rte_eth_dev_udp_tunnel_port_delete(uint16_t port_id,
2965                                    struct rte_eth_udp_tunnel *udp_tunnel)
2966 {
2967         struct rte_eth_dev *dev;
2968
2969         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2970         dev = &rte_eth_devices[port_id];
2971
2972         if (udp_tunnel == NULL) {
2973                 RTE_ETHDEV_LOG(ERR, "Invalid udp_tunnel parameter\n");
2974                 return -EINVAL;
2975         }
2976
2977         if (udp_tunnel->prot_type >= RTE_TUNNEL_TYPE_MAX) {
2978                 RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
2979                 return -EINVAL;
2980         }
2981
2982         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->udp_tunnel_port_del, -ENOTSUP);
2983         return eth_err(port_id, (*dev->dev_ops->udp_tunnel_port_del)(dev,
2984                                                                 udp_tunnel));
2985 }
2986
2987 int
2988 rte_eth_led_on(uint16_t port_id)
2989 {
2990         struct rte_eth_dev *dev;
2991
2992         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2993         dev = &rte_eth_devices[port_id];
2994         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_led_on, -ENOTSUP);
2995         return eth_err(port_id, (*dev->dev_ops->dev_led_on)(dev));
2996 }
2997
2998 int
2999 rte_eth_led_off(uint16_t port_id)
3000 {
3001         struct rte_eth_dev *dev;
3002
3003         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3004         dev = &rte_eth_devices[port_id];
3005         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_led_off, -ENOTSUP);
3006         return eth_err(port_id, (*dev->dev_ops->dev_led_off)(dev));
3007 }
3008
3009 /*
3010  * Returns index into MAC address array of addr. Use 00:00:00:00:00:00 to find
3011  * an empty spot.
3012  */
3013 static int
3014 get_mac_addr_index(uint16_t port_id, const struct ether_addr *addr)
3015 {
3016         struct rte_eth_dev_info dev_info;
3017         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
3018         unsigned i;
3019
3020         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3021         rte_eth_dev_info_get(port_id, &dev_info);
3022
3023         for (i = 0; i < dev_info.max_mac_addrs; i++)
3024                 if (memcmp(addr, &dev->data->mac_addrs[i], ETHER_ADDR_LEN) == 0)
3025                         return i;
3026
3027         return -1;
3028 }
3029
3030 static const struct ether_addr null_mac_addr;
3031
3032 int
3033 rte_eth_dev_mac_addr_add(uint16_t port_id, struct ether_addr *addr,
3034                         uint32_t pool)
3035 {
3036         struct rte_eth_dev *dev;
3037         int index;
3038         uint64_t pool_mask;
3039         int ret;
3040
3041         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3042         dev = &rte_eth_devices[port_id];
3043         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_add, -ENOTSUP);
3044
3045         if (is_zero_ether_addr(addr)) {
3046                 RTE_ETHDEV_LOG(ERR, "Port %u: Cannot add NULL MAC address\n",
3047                         port_id);
3048                 return -EINVAL;
3049         }
3050         if (pool >= ETH_64_POOLS) {
3051                 RTE_ETHDEV_LOG(ERR, "Pool id must be 0-%d\n", ETH_64_POOLS - 1);
3052                 return -EINVAL;
3053         }
3054
3055         index = get_mac_addr_index(port_id, addr);
3056         if (index < 0) {
3057                 index = get_mac_addr_index(port_id, &null_mac_addr);
3058                 if (index < 0) {
3059                         RTE_ETHDEV_LOG(ERR, "Port %u: MAC address array full\n",
3060                                 port_id);
3061                         return -ENOSPC;
3062                 }
3063         } else {
3064                 pool_mask = dev->data->mac_pool_sel[index];
3065
3066                 /* Check if both MAC address and pool is already there, and do nothing */
3067                 if (pool_mask & (1ULL << pool))
3068                         return 0;
3069         }
3070
3071         /* Update NIC */
3072         ret = (*dev->dev_ops->mac_addr_add)(dev, addr, index, pool);
3073
3074         if (ret == 0) {
3075                 /* Update address in NIC data structure */
3076                 ether_addr_copy(addr, &dev->data->mac_addrs[index]);
3077
3078                 /* Update pool bitmap in NIC data structure */
3079                 dev->data->mac_pool_sel[index] |= (1ULL << pool);
3080         }
3081
3082         return eth_err(port_id, ret);
3083 }
3084
3085 int
3086 rte_eth_dev_mac_addr_remove(uint16_t port_id, struct ether_addr *addr)
3087 {
3088         struct rte_eth_dev *dev;
3089         int index;
3090
3091         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3092         dev = &rte_eth_devices[port_id];
3093         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_remove, -ENOTSUP);
3094
3095         index = get_mac_addr_index(port_id, addr);
3096         if (index == 0) {
3097                 RTE_ETHDEV_LOG(ERR,
3098                         "Port %u: Cannot remove default MAC address\n",
3099                         port_id);
3100                 return -EADDRINUSE;
3101         } else if (index < 0)
3102                 return 0;  /* Do nothing if address wasn't found */
3103
3104         /* Update NIC */
3105         (*dev->dev_ops->mac_addr_remove)(dev, index);
3106
3107         /* Update address in NIC data structure */
3108         ether_addr_copy(&null_mac_addr, &dev->data->mac_addrs[index]);
3109
3110         /* reset pool bitmap */
3111         dev->data->mac_pool_sel[index] = 0;
3112
3113         return 0;
3114 }
3115
3116 int
3117 rte_eth_dev_default_mac_addr_set(uint16_t port_id, struct ether_addr *addr)
3118 {
3119         struct rte_eth_dev *dev;
3120         int ret;
3121
3122         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3123
3124         if (!is_valid_assigned_ether_addr(addr))
3125                 return -EINVAL;
3126
3127         dev = &rte_eth_devices[port_id];
3128         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_set, -ENOTSUP);
3129
3130         ret = (*dev->dev_ops->mac_addr_set)(dev, addr);
3131         if (ret < 0)
3132                 return ret;
3133
3134         /* Update default address in NIC data structure */
3135         ether_addr_copy(addr, &dev->data->mac_addrs[0]);
3136
3137         return 0;
3138 }
3139
3140
3141 /*
3142  * Returns index into MAC address array of addr. Use 00:00:00:00:00:00 to find
3143  * an empty spot.
3144  */
3145 static int
3146 get_hash_mac_addr_index(uint16_t port_id, const struct ether_addr *addr)
3147 {
3148         struct rte_eth_dev_info dev_info;
3149         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
3150         unsigned i;
3151
3152         rte_eth_dev_info_get(port_id, &dev_info);
3153         if (!dev->data->hash_mac_addrs)
3154                 return -1;
3155
3156         for (i = 0; i < dev_info.max_hash_mac_addrs; i++)
3157                 if (memcmp(addr, &dev->data->hash_mac_addrs[i],
3158                         ETHER_ADDR_LEN) == 0)
3159                         return i;
3160
3161         return -1;
3162 }
3163
3164 int
3165 rte_eth_dev_uc_hash_table_set(uint16_t port_id, struct ether_addr *addr,
3166                                 uint8_t on)
3167 {
3168         int index;
3169         int ret;
3170         struct rte_eth_dev *dev;
3171
3172         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3173
3174         dev = &rte_eth_devices[port_id];
3175         if (is_zero_ether_addr(addr)) {
3176                 RTE_ETHDEV_LOG(ERR, "Port %u: Cannot add NULL MAC address\n",
3177                         port_id);
3178                 return -EINVAL;
3179         }
3180
3181         index = get_hash_mac_addr_index(port_id, addr);
3182         /* Check if it's already there, and do nothing */
3183         if ((index >= 0) && on)
3184                 return 0;
3185
3186         if (index < 0) {
3187                 if (!on) {
3188                         RTE_ETHDEV_LOG(ERR,
3189                                 "Port %u: the MAC address was not set in UTA\n",
3190                                 port_id);
3191                         return -EINVAL;
3192                 }
3193
3194                 index = get_hash_mac_addr_index(port_id, &null_mac_addr);
3195                 if (index < 0) {
3196                         RTE_ETHDEV_LOG(ERR, "Port %u: MAC address array full\n",
3197                                 port_id);
3198                         return -ENOSPC;
3199                 }
3200         }
3201
3202         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->uc_hash_table_set, -ENOTSUP);
3203         ret = (*dev->dev_ops->uc_hash_table_set)(dev, addr, on);
3204         if (ret == 0) {
3205                 /* Update address in NIC data structure */
3206                 if (on)
3207                         ether_addr_copy(addr,
3208                                         &dev->data->hash_mac_addrs[index]);
3209                 else
3210                         ether_addr_copy(&null_mac_addr,
3211                                         &dev->data->hash_mac_addrs[index]);
3212         }
3213
3214         return eth_err(port_id, ret);
3215 }
3216
3217 int
3218 rte_eth_dev_uc_all_hash_table_set(uint16_t port_id, uint8_t on)
3219 {
3220         struct rte_eth_dev *dev;
3221
3222         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3223
3224         dev = &rte_eth_devices[port_id];
3225
3226         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->uc_all_hash_table_set, -ENOTSUP);
3227         return eth_err(port_id, (*dev->dev_ops->uc_all_hash_table_set)(dev,
3228                                                                        on));
3229 }
3230
3231 int rte_eth_set_queue_rate_limit(uint16_t port_id, uint16_t queue_idx,
3232                                         uint16_t tx_rate)
3233 {
3234         struct rte_eth_dev *dev;
3235         struct rte_eth_dev_info dev_info;
3236         struct rte_eth_link link;
3237
3238         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3239
3240         dev = &rte_eth_devices[port_id];
3241         rte_eth_dev_info_get(port_id, &dev_info);
3242         link = dev->data->dev_link;
3243
3244         if (queue_idx > dev_info.max_tx_queues) {
3245                 RTE_ETHDEV_LOG(ERR,
3246                         "Set queue rate limit:port %u: invalid queue id=%u\n",
3247                         port_id, queue_idx);
3248                 return -EINVAL;
3249         }
3250
3251         if (tx_rate > link.link_speed) {
3252                 RTE_ETHDEV_LOG(ERR,
3253                         "Set queue rate limit:invalid tx_rate=%u, bigger than link speed= %d\n",
3254                         tx_rate, link.link_speed);
3255                 return -EINVAL;
3256         }
3257
3258         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_queue_rate_limit, -ENOTSUP);
3259         return eth_err(port_id, (*dev->dev_ops->set_queue_rate_limit)(dev,
3260                                                         queue_idx, tx_rate));
3261 }
3262
3263 int
3264 rte_eth_mirror_rule_set(uint16_t port_id,
3265                         struct rte_eth_mirror_conf *mirror_conf,
3266                         uint8_t rule_id, uint8_t on)
3267 {
3268         struct rte_eth_dev *dev;
3269
3270         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3271         if (mirror_conf->rule_type == 0) {
3272                 RTE_ETHDEV_LOG(ERR, "Mirror rule type can not be 0\n");
3273                 return -EINVAL;
3274         }
3275
3276         if (mirror_conf->dst_pool >= ETH_64_POOLS) {
3277                 RTE_ETHDEV_LOG(ERR, "Invalid dst pool, pool id must be 0-%d\n",
3278                         ETH_64_POOLS - 1);
3279                 return -EINVAL;
3280         }
3281
3282         if ((mirror_conf->rule_type & (ETH_MIRROR_VIRTUAL_POOL_UP |
3283              ETH_MIRROR_VIRTUAL_POOL_DOWN)) &&
3284             (mirror_conf->pool_mask == 0)) {
3285                 RTE_ETHDEV_LOG(ERR,
3286                         "Invalid mirror pool, pool mask can not be 0\n");
3287                 return -EINVAL;
3288         }
3289
3290         if ((mirror_conf->rule_type & ETH_MIRROR_VLAN) &&
3291             mirror_conf->vlan.vlan_mask == 0) {
3292                 RTE_ETHDEV_LOG(ERR,
3293                         "Invalid vlan mask, vlan mask can not be 0\n");
3294                 return -EINVAL;
3295         }
3296
3297         dev = &rte_eth_devices[port_id];
3298         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mirror_rule_set, -ENOTSUP);
3299
3300         return eth_err(port_id, (*dev->dev_ops->mirror_rule_set)(dev,
3301                                                 mirror_conf, rule_id, on));
3302 }
3303
3304 int
3305 rte_eth_mirror_rule_reset(uint16_t port_id, uint8_t rule_id)
3306 {
3307         struct rte_eth_dev *dev;
3308
3309         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3310
3311         dev = &rte_eth_devices[port_id];
3312         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mirror_rule_reset, -ENOTSUP);
3313
3314         return eth_err(port_id, (*dev->dev_ops->mirror_rule_reset)(dev,
3315                                                                    rule_id));
3316 }
3317
3318 RTE_INIT(eth_dev_init_cb_lists)
3319 {
3320         int i;
3321
3322         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
3323                 TAILQ_INIT(&rte_eth_devices[i].link_intr_cbs);
3324 }
3325
3326 int
3327 rte_eth_dev_callback_register(uint16_t port_id,
3328                         enum rte_eth_event_type event,
3329                         rte_eth_dev_cb_fn cb_fn, void *cb_arg)
3330 {
3331         struct rte_eth_dev *dev;
3332         struct rte_eth_dev_callback *user_cb;
3333         uint32_t next_port; /* size is 32-bit to prevent loop wrap-around */
3334         uint16_t last_port;
3335
3336         if (!cb_fn)
3337                 return -EINVAL;
3338
3339         if (!rte_eth_dev_is_valid_port(port_id) && port_id != RTE_ETH_ALL) {
3340                 RTE_ETHDEV_LOG(ERR, "Invalid port_id=%d\n", port_id);
3341                 return -EINVAL;
3342         }
3343
3344         if (port_id == RTE_ETH_ALL) {
3345                 next_port = 0;
3346                 last_port = RTE_MAX_ETHPORTS - 1;
3347         } else {
3348                 next_port = last_port = port_id;
3349         }
3350
3351         rte_spinlock_lock(&rte_eth_dev_cb_lock);
3352
3353         do {
3354                 dev = &rte_eth_devices[next_port];
3355
3356                 TAILQ_FOREACH(user_cb, &(dev->link_intr_cbs), next) {
3357                         if (user_cb->cb_fn == cb_fn &&
3358                                 user_cb->cb_arg == cb_arg &&
3359                                 user_cb->event == event) {
3360                                 break;
3361                         }
3362                 }
3363
3364                 /* create a new callback. */
3365                 if (user_cb == NULL) {
3366                         user_cb = rte_zmalloc("INTR_USER_CALLBACK",
3367                                 sizeof(struct rte_eth_dev_callback), 0);
3368                         if (user_cb != NULL) {
3369                                 user_cb->cb_fn = cb_fn;
3370                                 user_cb->cb_arg = cb_arg;
3371                                 user_cb->event = event;
3372                                 TAILQ_INSERT_TAIL(&(dev->link_intr_cbs),
3373                                                   user_cb, next);
3374                         } else {
3375                                 rte_spinlock_unlock(&rte_eth_dev_cb_lock);
3376                                 rte_eth_dev_callback_unregister(port_id, event,
3377                                                                 cb_fn, cb_arg);
3378                                 return -ENOMEM;
3379                         }
3380
3381                 }
3382         } while (++next_port <= last_port);
3383
3384         rte_spinlock_unlock(&rte_eth_dev_cb_lock);
3385         return 0;
3386 }
3387
3388 int
3389 rte_eth_dev_callback_unregister(uint16_t port_id,
3390                         enum rte_eth_event_type event,
3391                         rte_eth_dev_cb_fn cb_fn, void *cb_arg)
3392 {
3393         int ret;
3394         struct rte_eth_dev *dev;
3395         struct rte_eth_dev_callback *cb, *next;
3396         uint32_t next_port; /* size is 32-bit to prevent loop wrap-around */
3397         uint16_t last_port;
3398
3399         if (!cb_fn)
3400                 return -EINVAL;
3401
3402         if (!rte_eth_dev_is_valid_port(port_id) && port_id != RTE_ETH_ALL) {
3403                 RTE_ETHDEV_LOG(ERR, "Invalid port_id=%d\n", port_id);
3404                 return -EINVAL;
3405         }
3406
3407         if (port_id == RTE_ETH_ALL) {
3408                 next_port = 0;
3409                 last_port = RTE_MAX_ETHPORTS - 1;
3410         } else {
3411                 next_port = last_port = port_id;
3412         }
3413
3414         rte_spinlock_lock(&rte_eth_dev_cb_lock);
3415
3416         do {
3417                 dev = &rte_eth_devices[next_port];
3418                 ret = 0;
3419                 for (cb = TAILQ_FIRST(&dev->link_intr_cbs); cb != NULL;
3420                      cb = next) {
3421
3422                         next = TAILQ_NEXT(cb, next);
3423
3424                         if (cb->cb_fn != cb_fn || cb->event != event ||
3425                             (cb->cb_arg != (void *)-1 && cb->cb_arg != cb_arg))
3426                                 continue;
3427
3428                         /*
3429                          * if this callback is not executing right now,
3430                          * then remove it.
3431                          */
3432                         if (cb->active == 0) {
3433                                 TAILQ_REMOVE(&(dev->link_intr_cbs), cb, next);
3434                                 rte_free(cb);
3435                         } else {
3436                                 ret = -EAGAIN;
3437                         }
3438                 }
3439         } while (++next_port <= last_port);
3440
3441         rte_spinlock_unlock(&rte_eth_dev_cb_lock);
3442         return ret;
3443 }
3444
3445 int
3446 _rte_eth_dev_callback_process(struct rte_eth_dev *dev,
3447         enum rte_eth_event_type event, void *ret_param)
3448 {
3449         struct rte_eth_dev_callback *cb_lst;
3450         struct rte_eth_dev_callback dev_cb;
3451         int rc = 0;
3452
3453         rte_spinlock_lock(&rte_eth_dev_cb_lock);
3454         TAILQ_FOREACH(cb_lst, &(dev->link_intr_cbs), next) {
3455                 if (cb_lst->cb_fn == NULL || cb_lst->event != event)
3456                         continue;
3457                 dev_cb = *cb_lst;
3458                 cb_lst->active = 1;
3459                 if (ret_param != NULL)
3460                         dev_cb.ret_param = ret_param;
3461
3462                 rte_spinlock_unlock(&rte_eth_dev_cb_lock);
3463                 rc = dev_cb.cb_fn(dev->data->port_id, dev_cb.event,
3464                                 dev_cb.cb_arg, dev_cb.ret_param);
3465                 rte_spinlock_lock(&rte_eth_dev_cb_lock);
3466                 cb_lst->active = 0;
3467         }
3468         rte_spinlock_unlock(&rte_eth_dev_cb_lock);
3469         return rc;
3470 }
3471
3472 void
3473 rte_eth_dev_probing_finish(struct rte_eth_dev *dev)
3474 {
3475         if (dev == NULL)
3476                 return;
3477
3478         _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_NEW, NULL);
3479
3480         dev->state = RTE_ETH_DEV_ATTACHED;
3481 }
3482
3483 int
3484 rte_eth_dev_rx_intr_ctl(uint16_t port_id, int epfd, int op, void *data)
3485 {
3486         uint32_t vec;
3487         struct rte_eth_dev *dev;
3488         struct rte_intr_handle *intr_handle;
3489         uint16_t qid;
3490         int rc;
3491
3492         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3493
3494         dev = &rte_eth_devices[port_id];
3495
3496         if (!dev->intr_handle) {
3497                 RTE_ETHDEV_LOG(ERR, "RX Intr handle unset\n");
3498                 return -ENOTSUP;
3499         }
3500
3501         intr_handle = dev->intr_handle;
3502         if (!intr_handle->intr_vec) {
3503                 RTE_ETHDEV_LOG(ERR, "RX Intr vector unset\n");
3504                 return -EPERM;
3505         }
3506
3507         for (qid = 0; qid < dev->data->nb_rx_queues; qid++) {
3508                 vec = intr_handle->intr_vec[qid];
3509                 rc = rte_intr_rx_ctl(intr_handle, epfd, op, vec, data);
3510                 if (rc && rc != -EEXIST) {
3511                         RTE_ETHDEV_LOG(ERR,
3512                                 "p %u q %u rx ctl error op %d epfd %d vec %u\n",
3513                                 port_id, qid, op, epfd, vec);
3514                 }
3515         }
3516
3517         return 0;
3518 }
3519
3520 int __rte_experimental
3521 rte_eth_dev_rx_intr_ctl_q_get_fd(uint16_t port_id, uint16_t queue_id)
3522 {
3523         struct rte_intr_handle *intr_handle;
3524         struct rte_eth_dev *dev;
3525         unsigned int efd_idx;
3526         uint32_t vec;
3527         int fd;
3528
3529         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -1);
3530
3531         dev = &rte_eth_devices[port_id];
3532
3533         if (queue_id >= dev->data->nb_rx_queues) {
3534                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", queue_id);
3535                 return -1;
3536         }
3537
3538         if (!dev->intr_handle) {
3539                 RTE_ETHDEV_LOG(ERR, "RX Intr handle unset\n");
3540                 return -1;
3541         }
3542
3543         intr_handle = dev->intr_handle;
3544         if (!intr_handle->intr_vec) {
3545                 RTE_ETHDEV_LOG(ERR, "RX Intr vector unset\n");
3546                 return -1;
3547         }
3548
3549         vec = intr_handle->intr_vec[queue_id];
3550         efd_idx = (vec >= RTE_INTR_VEC_RXTX_OFFSET) ?
3551                 (vec - RTE_INTR_VEC_RXTX_OFFSET) : vec;
3552         fd = intr_handle->efds[efd_idx];
3553
3554         return fd;
3555 }
3556
3557 const struct rte_memzone *
3558 rte_eth_dma_zone_reserve(const struct rte_eth_dev *dev, const char *ring_name,
3559                          uint16_t queue_id, size_t size, unsigned align,
3560                          int socket_id)
3561 {
3562         char z_name[RTE_MEMZONE_NAMESIZE];
3563         const struct rte_memzone *mz;
3564
3565         snprintf(z_name, sizeof(z_name), "eth_p%d_q%d_%s",
3566                  dev->data->port_id, queue_id, ring_name);
3567
3568         mz = rte_memzone_lookup(z_name);
3569         if (mz)
3570                 return mz;
3571
3572         return rte_memzone_reserve_aligned(z_name, size, socket_id,
3573                         RTE_MEMZONE_IOVA_CONTIG, align);
3574 }
3575
3576 int __rte_experimental
3577 rte_eth_dev_create(struct rte_device *device, const char *name,
3578         size_t priv_data_size,
3579         ethdev_bus_specific_init ethdev_bus_specific_init,
3580         void *bus_init_params,
3581         ethdev_init_t ethdev_init, void *init_params)
3582 {
3583         struct rte_eth_dev *ethdev;
3584         int retval;
3585
3586         RTE_FUNC_PTR_OR_ERR_RET(*ethdev_init, -EINVAL);
3587
3588         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
3589                 ethdev = rte_eth_dev_allocate(name);
3590                 if (!ethdev)
3591                         return -ENODEV;
3592
3593                 if (priv_data_size) {
3594                         ethdev->data->dev_private = rte_zmalloc_socket(
3595                                 name, priv_data_size, RTE_CACHE_LINE_SIZE,
3596                                 device->numa_node);
3597
3598                         if (!ethdev->data->dev_private) {
3599                                 RTE_LOG(ERR, EAL, "failed to allocate private data");
3600                                 retval = -ENOMEM;
3601                                 goto probe_failed;
3602                         }
3603                 }
3604         } else {
3605                 ethdev = rte_eth_dev_attach_secondary(name);
3606                 if (!ethdev) {
3607                         RTE_LOG(ERR, EAL, "secondary process attach failed, "
3608                                 "ethdev doesn't exist");
3609                         return  -ENODEV;
3610                 }
3611         }
3612
3613         ethdev->device = device;
3614
3615         if (ethdev_bus_specific_init) {
3616                 retval = ethdev_bus_specific_init(ethdev, bus_init_params);
3617                 if (retval) {
3618                         RTE_LOG(ERR, EAL,
3619                                 "ethdev bus specific initialisation failed");
3620                         goto probe_failed;
3621                 }
3622         }
3623
3624         retval = ethdev_init(ethdev, init_params);
3625         if (retval) {
3626                 RTE_LOG(ERR, EAL, "ethdev initialisation failed");
3627                 goto probe_failed;
3628         }
3629
3630         rte_eth_dev_probing_finish(ethdev);
3631
3632         return retval;
3633
3634 probe_failed:
3635         rte_eth_dev_release_port(ethdev);
3636         return retval;
3637 }
3638
3639 int  __rte_experimental
3640 rte_eth_dev_destroy(struct rte_eth_dev *ethdev,
3641         ethdev_uninit_t ethdev_uninit)
3642 {
3643         int ret;
3644
3645         ethdev = rte_eth_dev_allocated(ethdev->data->name);
3646         if (!ethdev)
3647                 return -ENODEV;
3648
3649         RTE_FUNC_PTR_OR_ERR_RET(*ethdev_uninit, -EINVAL);
3650         if (ethdev_uninit) {
3651                 ret = ethdev_uninit(ethdev);
3652                 if (ret)
3653                         return ret;
3654         }
3655
3656         return rte_eth_dev_release_port(ethdev);
3657 }
3658
3659 int
3660 rte_eth_dev_rx_intr_ctl_q(uint16_t port_id, uint16_t queue_id,
3661                           int epfd, int op, void *data)
3662 {
3663         uint32_t vec;
3664         struct rte_eth_dev *dev;
3665         struct rte_intr_handle *intr_handle;
3666         int rc;
3667
3668         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3669
3670         dev = &rte_eth_devices[port_id];
3671         if (queue_id >= dev->data->nb_rx_queues) {
3672                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", queue_id);
3673                 return -EINVAL;
3674         }
3675
3676         if (!dev->intr_handle) {
3677                 RTE_ETHDEV_LOG(ERR, "RX Intr handle unset\n");
3678                 return -ENOTSUP;
3679         }
3680
3681         intr_handle = dev->intr_handle;
3682         if (!intr_handle->intr_vec) {
3683                 RTE_ETHDEV_LOG(ERR, "RX Intr vector unset\n");
3684                 return -EPERM;
3685         }
3686
3687         vec = intr_handle->intr_vec[queue_id];
3688         rc = rte_intr_rx_ctl(intr_handle, epfd, op, vec, data);
3689         if (rc && rc != -EEXIST) {
3690                 RTE_ETHDEV_LOG(ERR,
3691                         "p %u q %u rx ctl error op %d epfd %d vec %u\n",
3692                         port_id, queue_id, op, epfd, vec);
3693                 return rc;
3694         }
3695
3696         return 0;
3697 }
3698
3699 int
3700 rte_eth_dev_rx_intr_enable(uint16_t port_id,
3701                            uint16_t queue_id)
3702 {
3703         struct rte_eth_dev *dev;
3704
3705         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3706
3707         dev = &rte_eth_devices[port_id];
3708
3709         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_intr_enable, -ENOTSUP);
3710         return eth_err(port_id, (*dev->dev_ops->rx_queue_intr_enable)(dev,
3711                                                                 queue_id));
3712 }
3713
3714 int
3715 rte_eth_dev_rx_intr_disable(uint16_t port_id,
3716                             uint16_t queue_id)
3717 {
3718         struct rte_eth_dev *dev;
3719
3720         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3721
3722         dev = &rte_eth_devices[port_id];
3723
3724         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_intr_disable, -ENOTSUP);
3725         return eth_err(port_id, (*dev->dev_ops->rx_queue_intr_disable)(dev,
3726                                                                 queue_id));
3727 }
3728
3729
3730 int
3731 rte_eth_dev_filter_supported(uint16_t port_id,
3732                              enum rte_filter_type filter_type)
3733 {
3734         struct rte_eth_dev *dev;
3735
3736         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3737
3738         dev = &rte_eth_devices[port_id];
3739         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->filter_ctrl, -ENOTSUP);
3740         return (*dev->dev_ops->filter_ctrl)(dev, filter_type,
3741                                 RTE_ETH_FILTER_NOP, NULL);
3742 }
3743
3744 int
3745 rte_eth_dev_filter_ctrl(uint16_t port_id, enum rte_filter_type filter_type,
3746                         enum rte_filter_op filter_op, void *arg)
3747 {
3748         struct rte_eth_dev *dev;
3749
3750         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3751
3752         dev = &rte_eth_devices[port_id];
3753         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->filter_ctrl, -ENOTSUP);
3754         return eth_err(port_id, (*dev->dev_ops->filter_ctrl)(dev, filter_type,
3755                                                              filter_op, arg));
3756 }
3757
3758 const struct rte_eth_rxtx_callback *
3759 rte_eth_add_rx_callback(uint16_t port_id, uint16_t queue_id,
3760                 rte_rx_callback_fn fn, void *user_param)
3761 {
3762 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
3763         rte_errno = ENOTSUP;
3764         return NULL;
3765 #endif
3766         /* check input parameters */
3767         if (!rte_eth_dev_is_valid_port(port_id) || fn == NULL ||
3768                     queue_id >= rte_eth_devices[port_id].data->nb_rx_queues) {
3769                 rte_errno = EINVAL;
3770                 return NULL;
3771         }
3772         struct rte_eth_rxtx_callback *cb = rte_zmalloc(NULL, sizeof(*cb), 0);
3773
3774         if (cb == NULL) {
3775                 rte_errno = ENOMEM;
3776                 return NULL;
3777         }
3778
3779         cb->fn.rx = fn;
3780         cb->param = user_param;
3781
3782         rte_spinlock_lock(&rte_eth_rx_cb_lock);
3783         /* Add the callbacks in fifo order. */
3784         struct rte_eth_rxtx_callback *tail =
3785                 rte_eth_devices[port_id].post_rx_burst_cbs[queue_id];
3786
3787         if (!tail) {
3788                 rte_eth_devices[port_id].post_rx_burst_cbs[queue_id] = cb;
3789
3790         } else {
3791                 while (tail->next)
3792                         tail = tail->next;
3793                 tail->next = cb;
3794         }
3795         rte_spinlock_unlock(&rte_eth_rx_cb_lock);
3796
3797         return cb;
3798 }
3799
3800 const struct rte_eth_rxtx_callback *
3801 rte_eth_add_first_rx_callback(uint16_t port_id, uint16_t queue_id,
3802                 rte_rx_callback_fn fn, void *user_param)
3803 {
3804 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
3805         rte_errno = ENOTSUP;
3806         return NULL;
3807 #endif
3808         /* check input parameters */
3809         if (!rte_eth_dev_is_valid_port(port_id) || fn == NULL ||
3810                 queue_id >= rte_eth_devices[port_id].data->nb_rx_queues) {
3811                 rte_errno = EINVAL;
3812                 return NULL;
3813         }
3814
3815         struct rte_eth_rxtx_callback *cb = rte_zmalloc(NULL, sizeof(*cb), 0);
3816
3817         if (cb == NULL) {
3818                 rte_errno = ENOMEM;
3819                 return NULL;
3820         }
3821
3822         cb->fn.rx = fn;
3823         cb->param = user_param;
3824
3825         rte_spinlock_lock(&rte_eth_rx_cb_lock);
3826         /* Add the callbacks at fisrt position*/
3827         cb->next = rte_eth_devices[port_id].post_rx_burst_cbs[queue_id];
3828         rte_smp_wmb();
3829         rte_eth_devices[port_id].post_rx_burst_cbs[queue_id] = cb;
3830         rte_spinlock_unlock(&rte_eth_rx_cb_lock);
3831
3832         return cb;
3833 }
3834
3835 const struct rte_eth_rxtx_callback *
3836 rte_eth_add_tx_callback(uint16_t port_id, uint16_t queue_id,
3837                 rte_tx_callback_fn fn, void *user_param)
3838 {
3839 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
3840         rte_errno = ENOTSUP;
3841         return NULL;
3842 #endif
3843         /* check input parameters */
3844         if (!rte_eth_dev_is_valid_port(port_id) || fn == NULL ||
3845                     queue_id >= rte_eth_devices[port_id].data->nb_tx_queues) {
3846                 rte_errno = EINVAL;
3847                 return NULL;
3848         }
3849
3850         struct rte_eth_rxtx_callback *cb = rte_zmalloc(NULL, sizeof(*cb), 0);
3851
3852         if (cb == NULL) {
3853                 rte_errno = ENOMEM;
3854                 return NULL;
3855         }
3856
3857         cb->fn.tx = fn;
3858         cb->param = user_param;
3859
3860         rte_spinlock_lock(&rte_eth_tx_cb_lock);
3861         /* Add the callbacks in fifo order. */
3862         struct rte_eth_rxtx_callback *tail =
3863                 rte_eth_devices[port_id].pre_tx_burst_cbs[queue_id];
3864
3865         if (!tail) {
3866                 rte_eth_devices[port_id].pre_tx_burst_cbs[queue_id] = cb;
3867
3868         } else {
3869                 while (tail->next)
3870                         tail = tail->next;
3871                 tail->next = cb;
3872         }
3873         rte_spinlock_unlock(&rte_eth_tx_cb_lock);
3874
3875         return cb;
3876 }
3877
3878 int
3879 rte_eth_remove_rx_callback(uint16_t port_id, uint16_t queue_id,
3880                 const struct rte_eth_rxtx_callback *user_cb)
3881 {
3882 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
3883         return -ENOTSUP;
3884 #endif
3885         /* Check input parameters. */
3886         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
3887         if (user_cb == NULL ||
3888                         queue_id >= rte_eth_devices[port_id].data->nb_rx_queues)
3889                 return -EINVAL;
3890
3891         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
3892         struct rte_eth_rxtx_callback *cb;
3893         struct rte_eth_rxtx_callback **prev_cb;
3894         int ret = -EINVAL;
3895
3896         rte_spinlock_lock(&rte_eth_rx_cb_lock);
3897         prev_cb = &dev->post_rx_burst_cbs[queue_id];
3898         for (; *prev_cb != NULL; prev_cb = &cb->next) {
3899                 cb = *prev_cb;
3900                 if (cb == user_cb) {
3901                         /* Remove the user cb from the callback list. */
3902                         *prev_cb = cb->next;
3903                         ret = 0;
3904                         break;
3905                 }
3906         }
3907         rte_spinlock_unlock(&rte_eth_rx_cb_lock);
3908
3909         return ret;
3910 }
3911
3912 int
3913 rte_eth_remove_tx_callback(uint16_t port_id, uint16_t queue_id,
3914                 const struct rte_eth_rxtx_callback *user_cb)
3915 {
3916 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
3917         return -ENOTSUP;
3918 #endif
3919         /* Check input parameters. */
3920         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
3921         if (user_cb == NULL ||
3922                         queue_id >= rte_eth_devices[port_id].data->nb_tx_queues)
3923                 return -EINVAL;
3924
3925         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
3926         int ret = -EINVAL;
3927         struct rte_eth_rxtx_callback *cb;
3928         struct rte_eth_rxtx_callback **prev_cb;
3929
3930         rte_spinlock_lock(&rte_eth_tx_cb_lock);
3931         prev_cb = &dev->pre_tx_burst_cbs[queue_id];
3932         for (; *prev_cb != NULL; prev_cb = &cb->next) {
3933                 cb = *prev_cb;
3934                 if (cb == user_cb) {
3935                         /* Remove the user cb from the callback list. */
3936                         *prev_cb = cb->next;
3937                         ret = 0;
3938                         break;
3939                 }
3940         }
3941         rte_spinlock_unlock(&rte_eth_tx_cb_lock);
3942
3943         return ret;
3944 }
3945
3946 int
3947 rte_eth_rx_queue_info_get(uint16_t port_id, uint16_t queue_id,
3948         struct rte_eth_rxq_info *qinfo)
3949 {
3950         struct rte_eth_dev *dev;
3951
3952         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3953
3954         if (qinfo == NULL)
3955                 return -EINVAL;
3956
3957         dev = &rte_eth_devices[port_id];
3958         if (queue_id >= dev->data->nb_rx_queues) {
3959                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", queue_id);
3960                 return -EINVAL;
3961         }
3962
3963         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rxq_info_get, -ENOTSUP);
3964
3965         memset(qinfo, 0, sizeof(*qinfo));
3966         dev->dev_ops->rxq_info_get(dev, queue_id, qinfo);
3967         return 0;
3968 }
3969
3970 int
3971 rte_eth_tx_queue_info_get(uint16_t port_id, uint16_t queue_id,
3972         struct rte_eth_txq_info *qinfo)
3973 {
3974         struct rte_eth_dev *dev;
3975
3976         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3977
3978         if (qinfo == NULL)
3979                 return -EINVAL;
3980
3981         dev = &rte_eth_devices[port_id];
3982         if (queue_id >= dev->data->nb_tx_queues) {
3983                 RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", queue_id);
3984                 return -EINVAL;
3985         }
3986
3987         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->txq_info_get, -ENOTSUP);
3988
3989         memset(qinfo, 0, sizeof(*qinfo));
3990         dev->dev_ops->txq_info_get(dev, queue_id, qinfo);
3991
3992         return 0;
3993 }
3994
3995 int
3996 rte_eth_dev_set_mc_addr_list(uint16_t port_id,
3997                              struct ether_addr *mc_addr_set,
3998                              uint32_t nb_mc_addr)
3999 {
4000         struct rte_eth_dev *dev;
4001
4002         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4003
4004         dev = &rte_eth_devices[port_id];
4005         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_mc_addr_list, -ENOTSUP);
4006         return eth_err(port_id, dev->dev_ops->set_mc_addr_list(dev,
4007                                                 mc_addr_set, nb_mc_addr));
4008 }
4009
4010 int
4011 rte_eth_timesync_enable(uint16_t port_id)
4012 {
4013         struct rte_eth_dev *dev;
4014
4015         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4016         dev = &rte_eth_devices[port_id];
4017
4018         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_enable, -ENOTSUP);
4019         return eth_err(port_id, (*dev->dev_ops->timesync_enable)(dev));
4020 }
4021
4022 int
4023 rte_eth_timesync_disable(uint16_t port_id)
4024 {
4025         struct rte_eth_dev *dev;
4026
4027         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4028         dev = &rte_eth_devices[port_id];
4029
4030         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_disable, -ENOTSUP);
4031         return eth_err(port_id, (*dev->dev_ops->timesync_disable)(dev));
4032 }
4033
4034 int
4035 rte_eth_timesync_read_rx_timestamp(uint16_t port_id, struct timespec *timestamp,
4036                                    uint32_t flags)
4037 {
4038         struct rte_eth_dev *dev;
4039
4040         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4041         dev = &rte_eth_devices[port_id];
4042
4043         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_rx_timestamp, -ENOTSUP);
4044         return eth_err(port_id, (*dev->dev_ops->timesync_read_rx_timestamp)
4045                                 (dev, timestamp, flags));
4046 }
4047
4048 int
4049 rte_eth_timesync_read_tx_timestamp(uint16_t port_id,
4050                                    struct timespec *timestamp)
4051 {
4052         struct rte_eth_dev *dev;
4053
4054         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4055         dev = &rte_eth_devices[port_id];
4056
4057         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_tx_timestamp, -ENOTSUP);
4058         return eth_err(port_id, (*dev->dev_ops->timesync_read_tx_timestamp)
4059                                 (dev, timestamp));
4060 }
4061
4062 int
4063 rte_eth_timesync_adjust_time(uint16_t port_id, int64_t delta)
4064 {
4065         struct rte_eth_dev *dev;
4066
4067         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4068         dev = &rte_eth_devices[port_id];
4069
4070         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_adjust_time, -ENOTSUP);
4071         return eth_err(port_id, (*dev->dev_ops->timesync_adjust_time)(dev,
4072                                                                       delta));
4073 }
4074
4075 int
4076 rte_eth_timesync_read_time(uint16_t port_id, struct timespec *timestamp)
4077 {
4078         struct rte_eth_dev *dev;
4079
4080         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4081         dev = &rte_eth_devices[port_id];
4082
4083         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_time, -ENOTSUP);
4084         return eth_err(port_id, (*dev->dev_ops->timesync_read_time)(dev,
4085                                                                 timestamp));
4086 }
4087
4088 int
4089 rte_eth_timesync_write_time(uint16_t port_id, const struct timespec *timestamp)
4090 {
4091         struct rte_eth_dev *dev;
4092
4093         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4094         dev = &rte_eth_devices[port_id];
4095
4096         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_write_time, -ENOTSUP);
4097         return eth_err(port_id, (*dev->dev_ops->timesync_write_time)(dev,
4098                                                                 timestamp));
4099 }
4100
4101 int
4102 rte_eth_dev_get_reg_info(uint16_t port_id, struct rte_dev_reg_info *info)
4103 {
4104         struct rte_eth_dev *dev;
4105
4106         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4107
4108         dev = &rte_eth_devices[port_id];
4109         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_reg, -ENOTSUP);
4110         return eth_err(port_id, (*dev->dev_ops->get_reg)(dev, info));
4111 }
4112
4113 int
4114 rte_eth_dev_get_eeprom_length(uint16_t port_id)
4115 {
4116         struct rte_eth_dev *dev;
4117
4118         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4119
4120         dev = &rte_eth_devices[port_id];
4121         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_eeprom_length, -ENOTSUP);
4122         return eth_err(port_id, (*dev->dev_ops->get_eeprom_length)(dev));
4123 }
4124
4125 int
4126 rte_eth_dev_get_eeprom(uint16_t port_id, struct rte_dev_eeprom_info *info)
4127 {
4128         struct rte_eth_dev *dev;
4129
4130         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4131
4132         dev = &rte_eth_devices[port_id];
4133         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_eeprom, -ENOTSUP);
4134         return eth_err(port_id, (*dev->dev_ops->get_eeprom)(dev, info));
4135 }
4136
4137 int
4138 rte_eth_dev_set_eeprom(uint16_t port_id, struct rte_dev_eeprom_info *info)
4139 {
4140         struct rte_eth_dev *dev;
4141
4142         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4143
4144         dev = &rte_eth_devices[port_id];
4145         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_eeprom, -ENOTSUP);
4146         return eth_err(port_id, (*dev->dev_ops->set_eeprom)(dev, info));
4147 }
4148
4149 int __rte_experimental
4150 rte_eth_dev_get_module_info(uint16_t port_id,
4151                             struct rte_eth_dev_module_info *modinfo)
4152 {
4153         struct rte_eth_dev *dev;
4154
4155         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4156
4157         dev = &rte_eth_devices[port_id];
4158         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_module_info, -ENOTSUP);
4159         return (*dev->dev_ops->get_module_info)(dev, modinfo);
4160 }
4161
4162 int __rte_experimental
4163 rte_eth_dev_get_module_eeprom(uint16_t port_id,
4164                               struct rte_dev_eeprom_info *info)
4165 {
4166         struct rte_eth_dev *dev;
4167
4168         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4169
4170         dev = &rte_eth_devices[port_id];
4171         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_module_eeprom, -ENOTSUP);
4172         return (*dev->dev_ops->get_module_eeprom)(dev, info);
4173 }
4174
4175 int
4176 rte_eth_dev_get_dcb_info(uint16_t port_id,
4177                              struct rte_eth_dcb_info *dcb_info)
4178 {
4179         struct rte_eth_dev *dev;
4180
4181         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4182
4183         dev = &rte_eth_devices[port_id];
4184         memset(dcb_info, 0, sizeof(struct rte_eth_dcb_info));
4185
4186         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_dcb_info, -ENOTSUP);
4187         return eth_err(port_id, (*dev->dev_ops->get_dcb_info)(dev, dcb_info));
4188 }
4189
4190 int
4191 rte_eth_dev_l2_tunnel_eth_type_conf(uint16_t port_id,
4192                                     struct rte_eth_l2_tunnel_conf *l2_tunnel)
4193 {
4194         struct rte_eth_dev *dev;
4195
4196         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4197         if (l2_tunnel == NULL) {
4198                 RTE_ETHDEV_LOG(ERR, "Invalid l2_tunnel parameter\n");
4199                 return -EINVAL;
4200         }
4201
4202         if (l2_tunnel->l2_tunnel_type >= RTE_TUNNEL_TYPE_MAX) {
4203                 RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
4204                 return -EINVAL;
4205         }
4206
4207         dev = &rte_eth_devices[port_id];
4208         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->l2_tunnel_eth_type_conf,
4209                                 -ENOTSUP);
4210         return eth_err(port_id, (*dev->dev_ops->l2_tunnel_eth_type_conf)(dev,
4211                                                                 l2_tunnel));
4212 }
4213
4214 int
4215 rte_eth_dev_l2_tunnel_offload_set(uint16_t port_id,
4216                                   struct rte_eth_l2_tunnel_conf *l2_tunnel,
4217                                   uint32_t mask,
4218                                   uint8_t en)
4219 {
4220         struct rte_eth_dev *dev;
4221
4222         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4223
4224         if (l2_tunnel == NULL) {
4225                 RTE_ETHDEV_LOG(ERR, "Invalid l2_tunnel parameter\n");
4226                 return -EINVAL;
4227         }
4228
4229         if (l2_tunnel->l2_tunnel_type >= RTE_TUNNEL_TYPE_MAX) {
4230                 RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
4231                 return -EINVAL;
4232         }
4233
4234         if (mask == 0) {
4235                 RTE_ETHDEV_LOG(ERR, "Mask should have a value\n");
4236                 return -EINVAL;
4237         }
4238
4239         dev = &rte_eth_devices[port_id];
4240         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->l2_tunnel_offload_set,
4241                                 -ENOTSUP);
4242         return eth_err(port_id, (*dev->dev_ops->l2_tunnel_offload_set)(dev,
4243                                                         l2_tunnel, mask, en));
4244 }
4245
4246 static void
4247 rte_eth_dev_adjust_nb_desc(uint16_t *nb_desc,
4248                            const struct rte_eth_desc_lim *desc_lim)
4249 {
4250         if (desc_lim->nb_align != 0)
4251                 *nb_desc = RTE_ALIGN_CEIL(*nb_desc, desc_lim->nb_align);
4252
4253         if (desc_lim->nb_max != 0)
4254                 *nb_desc = RTE_MIN(*nb_desc, desc_lim->nb_max);
4255
4256         *nb_desc = RTE_MAX(*nb_desc, desc_lim->nb_min);
4257 }
4258
4259 int
4260 rte_eth_dev_adjust_nb_rx_tx_desc(uint16_t port_id,
4261                                  uint16_t *nb_rx_desc,
4262                                  uint16_t *nb_tx_desc)
4263 {
4264         struct rte_eth_dev *dev;
4265         struct rte_eth_dev_info dev_info;
4266
4267         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4268
4269         dev = &rte_eth_devices[port_id];
4270         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
4271
4272         rte_eth_dev_info_get(port_id, &dev_info);
4273
4274         if (nb_rx_desc != NULL)
4275                 rte_eth_dev_adjust_nb_desc(nb_rx_desc, &dev_info.rx_desc_lim);
4276
4277         if (nb_tx_desc != NULL)
4278                 rte_eth_dev_adjust_nb_desc(nb_tx_desc, &dev_info.tx_desc_lim);
4279
4280         return 0;
4281 }
4282
4283 int
4284 rte_eth_dev_pool_ops_supported(uint16_t port_id, const char *pool)
4285 {
4286         struct rte_eth_dev *dev;
4287
4288         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4289
4290         if (pool == NULL)
4291                 return -EINVAL;
4292
4293         dev = &rte_eth_devices[port_id];
4294
4295         if (*dev->dev_ops->pool_ops_supported == NULL)
4296                 return 1; /* all pools are supported */
4297
4298         return (*dev->dev_ops->pool_ops_supported)(dev, pool);
4299 }
4300
4301 /**
4302  * A set of values to describe the possible states of a switch domain.
4303  */
4304 enum rte_eth_switch_domain_state {
4305         RTE_ETH_SWITCH_DOMAIN_UNUSED = 0,
4306         RTE_ETH_SWITCH_DOMAIN_ALLOCATED
4307 };
4308
4309 /**
4310  * Array of switch domains available for allocation. Array is sized to
4311  * RTE_MAX_ETHPORTS elements as there cannot be more active switch domains than
4312  * ethdev ports in a single process.
4313  */
4314 static struct rte_eth_dev_switch {
4315         enum rte_eth_switch_domain_state state;
4316 } rte_eth_switch_domains[RTE_MAX_ETHPORTS];
4317
4318 int __rte_experimental
4319 rte_eth_switch_domain_alloc(uint16_t *domain_id)
4320 {
4321         unsigned int i;
4322
4323         *domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID;
4324
4325         for (i = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID + 1;
4326                 i < RTE_MAX_ETHPORTS; i++) {
4327                 if (rte_eth_switch_domains[i].state ==
4328                         RTE_ETH_SWITCH_DOMAIN_UNUSED) {
4329                         rte_eth_switch_domains[i].state =
4330                                 RTE_ETH_SWITCH_DOMAIN_ALLOCATED;
4331                         *domain_id = i;
4332                         return 0;
4333                 }
4334         }
4335
4336         return -ENOSPC;
4337 }
4338
4339 int __rte_experimental
4340 rte_eth_switch_domain_free(uint16_t domain_id)
4341 {
4342         if (domain_id == RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID ||
4343                 domain_id >= RTE_MAX_ETHPORTS)
4344                 return -EINVAL;
4345
4346         if (rte_eth_switch_domains[domain_id].state !=
4347                 RTE_ETH_SWITCH_DOMAIN_ALLOCATED)
4348                 return -EINVAL;
4349
4350         rte_eth_switch_domains[domain_id].state = RTE_ETH_SWITCH_DOMAIN_UNUSED;
4351
4352         return 0;
4353 }
4354
4355 static int
4356 rte_eth_devargs_tokenise(struct rte_kvargs *arglist, const char *str_in)
4357 {
4358         int state;
4359         struct rte_kvargs_pair *pair;
4360         char *letter;
4361
4362         arglist->str = strdup(str_in);
4363         if (arglist->str == NULL)
4364                 return -ENOMEM;
4365
4366         letter = arglist->str;
4367         state = 0;
4368         arglist->count = 0;
4369         pair = &arglist->pairs[0];
4370         while (1) {
4371                 switch (state) {
4372                 case 0: /* Initial */
4373                         if (*letter == '=')
4374                                 return -EINVAL;
4375                         else if (*letter == '\0')
4376                                 return 0;
4377
4378                         state = 1;
4379                         pair->key = letter;
4380                         /* fall-thru */
4381
4382                 case 1: /* Parsing key */
4383                         if (*letter == '=') {
4384                                 *letter = '\0';
4385                                 pair->value = letter + 1;
4386                                 state = 2;
4387                         } else if (*letter == ',' || *letter == '\0')
4388                                 return -EINVAL;
4389                         break;
4390
4391
4392                 case 2: /* Parsing value */
4393                         if (*letter == '[')
4394                                 state = 3;
4395                         else if (*letter == ',') {
4396                                 *letter = '\0';
4397                                 arglist->count++;
4398                                 pair = &arglist->pairs[arglist->count];
4399                                 state = 0;
4400                         } else if (*letter == '\0') {
4401                                 letter--;
4402                                 arglist->count++;
4403                                 pair = &arglist->pairs[arglist->count];
4404                                 state = 0;
4405                         }
4406                         break;
4407
4408                 case 3: /* Parsing list */
4409                         if (*letter == ']')
4410                                 state = 2;
4411                         else if (*letter == '\0')
4412                                 return -EINVAL;
4413                         break;
4414                 }
4415                 letter++;
4416         }
4417 }
4418
4419 int __rte_experimental
4420 rte_eth_devargs_parse(const char *dargs, struct rte_eth_devargs *eth_da)
4421 {
4422         struct rte_kvargs args;
4423         struct rte_kvargs_pair *pair;
4424         unsigned int i;
4425         int result = 0;
4426
4427         memset(eth_da, 0, sizeof(*eth_da));
4428
4429         result = rte_eth_devargs_tokenise(&args, dargs);
4430         if (result < 0)
4431                 goto parse_cleanup;
4432
4433         for (i = 0; i < args.count; i++) {
4434                 pair = &args.pairs[i];
4435                 if (strcmp("representor", pair->key) == 0) {
4436                         result = rte_eth_devargs_parse_list(pair->value,
4437                                 rte_eth_devargs_parse_representor_ports,
4438                                 eth_da);
4439                         if (result < 0)
4440                                 goto parse_cleanup;
4441                 }
4442         }
4443
4444 parse_cleanup:
4445         if (args.str)
4446                 free(args.str);
4447
4448         return result;
4449 }
4450
4451 RTE_INIT(ethdev_init_log)
4452 {
4453         rte_eth_dev_logtype = rte_log_register("lib.ethdev");
4454         if (rte_eth_dev_logtype >= 0)
4455                 rte_log_set_level(rte_eth_dev_logtype, RTE_LOG_INFO);
4456 }