app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_malloc_heap.h>
  31 #include <rte_memory.h>
  32 #include <rte_memcpy.h>
  33 #include <rte_launch.h>
  34 #include <rte_eal.h>
  35 #include <rte_alarm.h>
  36 #include <rte_per_lcore.h>
  37 #include <rte_lcore.h>
  38 #include <rte_atomic.h>
  39 #include <rte_branch_prediction.h>
  40 #include <rte_mempool.h>
  41 #include <rte_malloc.h>
  42 #include <rte_mbuf.h>
  43 #include <rte_mbuf_pool_ops.h>
  44 #include <rte_interrupts.h>
  45 #include <rte_pci.h>
  46 #include <rte_ether.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_dev.h>
  49 #include <rte_string_fns.h>
  50 #ifdef RTE_LIBRTE_IXGBE_PMD
  51 #include <rte_pmd_ixgbe.h>
  52 #endif
  53 #ifdef RTE_LIBRTE_PDUMP
  54 #include <rte_pdump.h>
  55 #endif
  56 #include <rte_flow.h>
  57 #include <rte_metrics.h>
  58 #ifdef RTE_LIBRTE_BITRATE
  59 #include <rte_bitrate.h>
  60 #endif
  61 #ifdef RTE_LIBRTE_LATENCY_STATS
  62 #include <rte_latencystats.h>
  63 #endif
  64
  65 #include "testpmd.h"
  66
  67 #ifndef MAP_HUGETLB
  68 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  69 #define HUGE_FLAG (0x40000)
  70 #else
  71 #define HUGE_FLAG MAP_HUGETLB
  72 #endif
  73
  74 #ifndef MAP_HUGE_SHIFT
  75 /* older kernels (or FreeBSD) will not have this define */
  76 #define HUGE_SHIFT (26)
  77 #else
  78 #define HUGE_SHIFT MAP_HUGE_SHIFT
  79 #endif
  80
  81 #define EXTMEM_HEAP_NAME "extmem"
  82
  83 uint16_t verbose_level = 0; /**< Silent by default. */
  84 int testpmd_logtype; /**< Log type for testpmd logs */
  85
  86 /* use master core for command line ? */
  87 uint8_t interactive = 0;
  88 uint8_t auto_start = 0;
  89 uint8_t tx_first;
  90 char cmdline_filename[PATH_MAX] = {0};
  91
  92 /*
  93  * NUMA support configuration.
  94  * When set, the NUMA support attempts to dispatch the allocation of the
  95  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  96  * probed ports among the CPU sockets 0 and 1.
  97  * Otherwise, all memory is allocated from CPU socket 0.
  98  */
  99 uint8_t numa_support = 1; /**< numa enabled by default */
 100
 101 /*
 102  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 103  * not configured.
 104  */
 105 uint8_t socket_num = UMA_NO_CONFIG;
 106
 107 /*
 108  * Select mempool allocation type:
 109  * - native: use regular DPDK memory
 110  * - anon: use regular DPDK memory to create mempool, but populate using
 111  *         anonymous memory (may not be IOVA-contiguous)
 112  * - xmem: use externally allocated hugepage memory
 113  */
 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 115
 116 /*
 117  * Store specified sockets on which memory pool to be used by ports
 118  * is allocated.
 119  */
 120 uint8_t port_numa[RTE_MAX_ETHPORTS];
 121
 122 /*
 123  * Store specified sockets on which RX ring to be used by ports
 124  * is allocated.
 125  */
 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 127
 128 /*
 129  * Store specified sockets on which TX ring to be used by ports
 130  * is allocated.
 131  */
 132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 133
 134 /*
 135  * Record the Ethernet address of peer target ports to which packets are
 136  * forwarded.
 137  * Must be instantiated with the ethernet addresses of peer traffic generator
 138  * ports.
 139  */
 140 struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 141 portid_t nb_peer_eth_addrs = 0;
 142
 143 /*
 144  * Probed Target Environment.
 145  */
 146 struct rte_port *ports;        /**< For all probed ethernet ports. */
 147 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 149 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 150
 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 152
 153 /*
 154  * Test Forwarding Configuration.
 155  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 156  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 157  */
 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 160 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 161 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 162
 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 165
 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 167 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 168
 169 /*
 170  * Forwarding engines.
 171  */
 172 struct fwd_engine * fwd_engines[] = {
 173         &io_fwd_engine,
 174         &mac_fwd_engine,
 175         &mac_swap_engine,
 176         &flow_gen_engine,
 177         &rx_only_engine,
 178         &tx_only_engine,
 179         &csum_fwd_engine,
 180         &icmp_echo_engine,
 181         &noisy_vnf_engine,
 182 #if defined RTE_LIBRTE_PMD_SOFTNIC
 183         &softnic_fwd_engine,
 184 #endif
 185 #ifdef RTE_LIBRTE_IEEE1588
 186         &ieee1588_fwd_engine,
 187 #endif
 188         NULL,
 189 };
 190
 191 struct fwd_config cur_fwd_config;
 192 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 193 uint32_t retry_enabled;
 194 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 195 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 196
 197 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
 198 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 199                                       * specified on command-line. */
 200 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 201
 202 /*
 203  * In container, it cannot terminate the process which running with 'stats-period'
 204  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 205  */
 206 uint8_t f_quit;
 207
 208 /*
 209  * Configuration of packet segments used by the "txonly" processing engine.
 210  */
 211 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 212 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 213         TXONLY_DEF_PACKET_LEN,
 214 };
 215 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 216
 217 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 218 /**< Split policy for packets to TX. */
 219
 220 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 221 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 222
 223 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 224 uint8_t dcb_config = 0;
 225
 226 /* Whether the dcb is in testing status */
 227 uint8_t dcb_test = 0;
 228
 229 /*
 230  * Configurable number of RX/TX queues.
 231  */
 232 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 233 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 234
 235 /*
 236  * Configurable number of RX/TX ring descriptors.
 237  * Defaults are supplied by drivers via ethdev.
 238  */
 239 #define RTE_TEST_RX_DESC_DEFAULT 0
 240 #define RTE_TEST_TX_DESC_DEFAULT 0
 241 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 242 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 243
 244 #define RTE_PMD_PARAM_UNSET -1
 245 /*
 246  * Configurable values of RX and TX ring threshold registers.
 247  */
 248
 249 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 250 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 251 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 252
 253 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 254 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 255 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 256
 257 /*
 258  * Configurable value of RX free threshold.
 259  */
 260 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 261
 262 /*
 263  * Configurable value of RX drop enable.
 264  */
 265 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 266
 267 /*
 268  * Configurable value of TX free threshold.
 269  */
 270 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 271
 272 /*
 273  * Configurable value of TX RS bit threshold.
 274  */
 275 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 276
 277 /*
 278  * Configurable value of buffered packets before sending.
 279  */
 280 uint16_t noisy_tx_sw_bufsz;
 281
 282 /*
 283  * Configurable value of packet buffer timeout.
 284  */
 285 uint16_t noisy_tx_sw_buf_flush_time;
 286
 287 /*
 288  * Configurable value for size of VNF internal memory area
 289  * used for simulating noisy neighbour behaviour
 290  */
 291 uint64_t noisy_lkup_mem_sz;
 292
 293 /*
 294  * Configurable value of number of random writes done in
 295  * VNF simulation memory area.
 296  */
 297 uint64_t noisy_lkup_num_writes;
 298
 299 /*
 300  * Configurable value of number of random reads done in
 301  * VNF simulation memory area.
 302  */
 303 uint64_t noisy_lkup_num_reads;
 304
 305 /*
 306  * Configurable value of number of random reads/writes done in
 307  * VNF simulation memory area.
 308  */
 309 uint64_t noisy_lkup_num_reads_writes;
 310
 311 /*
 312  * Receive Side Scaling (RSS) configuration.
 313  */
 314 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 315
 316 /*
 317  * Port topology configuration
 318  */
 319 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 320
 321 /*
 322  * Avoids to flush all the RX streams before starts forwarding.
 323  */
 324 uint8_t no_flush_rx = 0; /* flush by default */
 325
 326 /*
 327  * Flow API isolated mode.
 328  */
 329 uint8_t flow_isolate_all;
 330
 331 /*
 332  * Avoids to check link status when starting/stopping a port.
 333  */
 334 uint8_t no_link_check = 0; /* check by default */
 335
 336 /*
 337  * Enable link status change notification
 338  */
 339 uint8_t lsc_interrupt = 1; /* enabled by default */
 340
 341 /*
 342  * Enable device removal notification.
 343  */
 344 uint8_t rmv_interrupt = 1; /* enabled by default */
 345
 346 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 347
 348 /* After attach, port setup is called on event or by iterator */
 349 bool setup_on_probe_event = true;
 350
 351 /* Pretty printing of ethdev events */
 352 static const char * const eth_event_desc[] = {
 353         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 354         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 355         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 356         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 357         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 358         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 359         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 360         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 361         [RTE_ETH_EVENT_NEW] = "device probed",
 362         [RTE_ETH_EVENT_DESTROY] = "device released",
 363         [RTE_ETH_EVENT_MAX] = NULL,
 364 };
 365
 366 /*
 367  * Display or mask ether events
 368  * Default to all events except VF_MBOX
 369  */
 370 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 371                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 372                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 373                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 374                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 375                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 376                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV);
 377 /*
 378  * Decide if all memory are locked for performance.
 379  */
 380 int do_mlockall = 0;
 381
 382 /*
 383  * NIC bypass mode configuration options.
 384  */
 385
 386 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
 387 /* The NIC bypass watchdog timeout. */
 388 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 389 #endif
 390
 391
 392 #ifdef RTE_LIBRTE_LATENCY_STATS
 393
 394 /*
 395  * Set when latency stats is enabled in the commandline
 396  */
 397 uint8_t latencystats_enabled;
 398
 399 /*
 400  * Lcore ID to serive latency statistics.
 401  */
 402 lcoreid_t latencystats_lcore_id = -1;
 403
 404 #endif
 405
 406 /*
 407  * Ethernet device configuration.
 408  */
 409 struct rte_eth_rxmode rx_mode = {
 410         .max_rx_pkt_len = ETHER_MAX_LEN, /**< Default maximum frame length. */
 411 };
 412
 413 struct rte_eth_txmode tx_mode = {
 414         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 415 };
 416
 417 struct rte_fdir_conf fdir_conf = {
 418         .mode = RTE_FDIR_MODE_NONE,
 419         .pballoc = RTE_FDIR_PBALLOC_64K,
 420         .status = RTE_FDIR_REPORT_STATUS,
 421         .mask = {
 422                 .vlan_tci_mask = 0xFFEF,
 423                 .ipv4_mask     = {
 424                         .src_ip = 0xFFFFFFFF,
 425                         .dst_ip = 0xFFFFFFFF,
 426                 },
 427                 .ipv6_mask     = {
 428                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 429                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 430                 },
 431                 .src_port_mask = 0xFFFF,
 432                 .dst_port_mask = 0xFFFF,
 433                 .mac_addr_byte_mask = 0xFF,
 434                 .tunnel_type_mask = 1,
 435                 .tunnel_id_mask = 0xFFFFFFFF,
 436         },
 437         .drop_queue = 127,
 438 };
 439
 440 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 441
 442 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 443 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 444
 445 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 446 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 447
 448 uint16_t nb_tx_queue_stats_mappings = 0;
 449 uint16_t nb_rx_queue_stats_mappings = 0;
 450
 451 /*
 452  * Display zero values by default for xstats
 453  */
 454 uint8_t xstats_hide_zero;
 455
 456 unsigned int num_sockets = 0;
 457 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 458
 459 #ifdef RTE_LIBRTE_BITRATE
 460 /* Bitrate statistics */
 461 struct rte_stats_bitrates *bitrate_data;
 462 lcoreid_t bitrate_lcore_id;
 463 uint8_t bitrate_enabled;
 464 #endif
 465
 466 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 467 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 468
 469 struct vxlan_encap_conf vxlan_encap_conf = {
 470         .select_ipv4 = 1,
 471         .select_vlan = 0,
 472         .vni = "\x00\x00\x00",
 473         .udp_src = 0,
 474         .udp_dst = RTE_BE16(4789),
 475         .ipv4_src = IPv4(127, 0, 0, 1),
 476         .ipv4_dst = IPv4(255, 255, 255, 255),
 477         .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00"
 478                 "\x00\x00\x00\x00\x00\x00\x00\x01",
 479         .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00"
 480                 "\x00\x00\x00\x00\x00\x00\x11\x11",
 481         .vlan_tci = 0,
 482         .eth_src = "\x00\x00\x00\x00\x00\x00",
 483         .eth_dst = "\xff\xff\xff\xff\xff\xff",
 484 };
 485
 486 struct nvgre_encap_conf nvgre_encap_conf = {
 487         .select_ipv4 = 1,
 488         .select_vlan = 0,
 489         .tni = "\x00\x00\x00",
 490         .ipv4_src = IPv4(127, 0, 0, 1),
 491         .ipv4_dst = IPv4(255, 255, 255, 255),
 492         .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00"
 493                 "\x00\x00\x00\x00\x00\x00\x00\x01",
 494         .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00"
 495                 "\x00\x00\x00\x00\x00\x00\x11\x11",
 496         .vlan_tci = 0,
 497         .eth_src = "\x00\x00\x00\x00\x00\x00",
 498         .eth_dst = "\xff\xff\xff\xff\xff\xff",
 499 };
 500
 501 /* Forward function declarations */
 502 static void setup_attached_port(portid_t pi);
 503 static void map_port_queue_stats_mapping_registers(portid_t pi,
 504                                                    struct rte_port *port);
 505 static void check_all_ports_link_status(uint32_t port_mask);
 506 static int eth_event_callback(portid_t port_id,
 507                               enum rte_eth_event_type type,
 508                               void *param, void *ret_param);
 509 static void eth_dev_event_callback(const char *device_name,
 510                                 enum rte_dev_event_type type,
 511                                 void *param);
 512
 513 /*
 514  * Check if all the ports are started.
 515  * If yes, return positive value. If not, return zero.
 516  */
 517 static int all_ports_started(void);
 518
 519 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 520 uint16_t gso_max_segment_size = ETHER_MAX_LEN - ETHER_CRC_LEN;
 521
 522 /*
 523  * Helper function to check if socket is already discovered.
 524  * If yes, return positive value. If not, return zero.
 525  */
 526 int
 527 new_socket_id(unsigned int socket_id)
 528 {
 529         unsigned int i;
 530
 531         for (i = 0; i < num_sockets; i++) {
 532                 if (socket_ids[i] == socket_id)
 533                         return 0;
 534         }
 535         return 1;
 536 }
 537
 538 /*
 539  * Setup default configuration.
 540  */
 541 static void
 542 set_default_fwd_lcores_config(void)
 543 {
 544         unsigned int i;
 545         unsigned int nb_lc;
 546         unsigned int sock_num;
 547
 548         nb_lc = 0;
 549         for (i = 0; i < RTE_MAX_LCORE; i++) {
 550                 if (!rte_lcore_is_enabled(i))
 551                         continue;
 552                 sock_num = rte_lcore_to_socket_id(i);
 553                 if (new_socket_id(sock_num)) {
 554                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 555                                 rte_exit(EXIT_FAILURE,
 556                                          "Total sockets greater than %u\n",
 557                                          RTE_MAX_NUMA_NODES);
 558                         }
 559                         socket_ids[num_sockets++] = sock_num;
 560                 }
 561                 if (i == rte_get_master_lcore())
 562                         continue;
 563                 fwd_lcores_cpuids[nb_lc++] = i;
 564         }
 565         nb_lcores = (lcoreid_t) nb_lc;
 566         nb_cfg_lcores = nb_lcores;
 567         nb_fwd_lcores = 1;
 568 }
 569
 570 static void
 571 set_def_peer_eth_addrs(void)
 572 {
 573         portid_t i;
 574
 575         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 576                 peer_eth_addrs[i].addr_bytes[0] = ETHER_LOCAL_ADMIN_ADDR;
 577                 peer_eth_addrs[i].addr_bytes[5] = i;
 578         }
 579 }
 580
 581 static void
 582 set_default_fwd_ports_config(void)
 583 {
 584         portid_t pt_id;
 585         int i = 0;
 586
 587         RTE_ETH_FOREACH_DEV(pt_id) {
 588                 fwd_ports_ids[i++] = pt_id;
 589
 590                 /* Update sockets info according to the attached device */
 591                 int socket_id = rte_eth_dev_socket_id(pt_id);
 592                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 593                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 594                                 rte_exit(EXIT_FAILURE,
 595                                          "Total sockets greater than %u\n",
 596                                          RTE_MAX_NUMA_NODES);
 597                         }
 598                         socket_ids[num_sockets++] = socket_id;
 599                 }
 600         }
 601
 602         nb_cfg_ports = nb_ports;
 603         nb_fwd_ports = nb_ports;
 604 }
 605
 606 void
 607 set_def_fwd_config(void)
 608 {
 609         set_default_fwd_lcores_config();
 610         set_def_peer_eth_addrs();
 611         set_default_fwd_ports_config();
 612 }
 613
 614 /* extremely pessimistic estimation of memory required to create a mempool */
 615 static int
 616 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 617 {
 618         unsigned int n_pages, mbuf_per_pg, leftover;
 619         uint64_t total_mem, mbuf_mem, obj_sz;
 620
 621         /* there is no good way to predict how much space the mempool will
 622          * occupy because it will allocate chunks on the fly, and some of those
 623          * will come from default DPDK memory while some will come from our
 624          * external memory, so just assume 128MB will be enough for everyone.
 625          */
 626         uint64_t hdr_mem = 128 << 20;
 627
 628         /* account for possible non-contiguousness */
 629         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 630         if (obj_sz > pgsz) {
 631                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 632                 return -1;
 633         }
 634
 635         mbuf_per_pg = pgsz / obj_sz;
 636         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 637         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 638
 639         mbuf_mem = n_pages * pgsz;
 640
 641         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 642
 643         if (total_mem > SIZE_MAX) {
 644                 TESTPMD_LOG(ERR, "Memory size too big\n");
 645                 return -1;
 646         }
 647         *out = (size_t)total_mem;
 648
 649         return 0;
 650 }
 651
 652 static inline uint32_t
 653 bsf64(uint64_t v)
 654 {
 655         return (uint32_t)__builtin_ctzll(v);
 656 }
 657
 658 static inline uint32_t
 659 log2_u64(uint64_t v)
 660 {
 661         if (v == 0)
 662                 return 0;
 663         v = rte_align64pow2(v);
 664         return bsf64(v);
 665 }
 666
 667 static int
 668 pagesz_flags(uint64_t page_sz)
 669 {
 670         /* as per mmap() manpage, all page sizes are log2 of page size
 671          * shifted by MAP_HUGE_SHIFT
 672          */
 673         int log2 = log2_u64(page_sz);
 674
 675         return (log2 << HUGE_SHIFT);
 676 }
 677
 678 static void *
 679 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 680 {
 681         void *addr;
 682         int flags;
 683
 684         /* allocate anonymous hugepages */
 685         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 686         if (huge)
 687                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 688
 689         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 690         if (addr == MAP_FAILED)
 691                 return NULL;
 692
 693         return addr;
 694 }
 695
 696 struct extmem_param {
 697         void *addr;
 698         size_t len;
 699         size_t pgsz;
 700         rte_iova_t *iova_table;
 701         unsigned int iova_table_len;
 702 };
 703
 704 static int
 705 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 706                 bool huge)
 707 {
 708         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 709                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 710         unsigned int cur_page, n_pages, pgsz_idx;
 711         size_t mem_sz, cur_pgsz;
 712         rte_iova_t *iovas = NULL;
 713         void *addr;
 714         int ret;
 715
 716         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 717                 /* skip anything that is too big */
 718                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 719                         continue;
 720
 721                 cur_pgsz = pgsizes[pgsz_idx];
 722
 723                 /* if we were told not to allocate hugepages, override */
 724                 if (!huge)
 725                         cur_pgsz = sysconf(_SC_PAGESIZE);
 726
 727                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 728                 if (ret < 0) {
 729                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 730                         return -1;
 731                 }
 732
 733                 /* allocate our memory */
 734                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 735
 736                 /* if we couldn't allocate memory with a specified page size,
 737                  * that doesn't mean we can't do it with other page sizes, so
 738                  * try another one.
 739                  */
 740                 if (addr == NULL)
 741                         continue;
 742
 743                 /* store IOVA addresses for every page in this memory area */
 744                 n_pages = mem_sz / cur_pgsz;
 745
 746                 iovas = malloc(sizeof(*iovas) * n_pages);
 747
 748                 if (iovas == NULL) {
 749                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 750                         goto fail;
 751                 }
 752                 /* lock memory if it's not huge pages */
 753                 if (!huge)
 754                         mlock(addr, mem_sz);
 755
 756                 /* populate IOVA addresses */
 757                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 758                         rte_iova_t iova;
 759                         size_t offset;
 760                         void *cur;
 761
 762                         offset = cur_pgsz * cur_page;
 763                         cur = RTE_PTR_ADD(addr, offset);
 764
 765                         /* touch the page before getting its IOVA */
 766                         *(volatile char *)cur = 0;
 767
 768                         iova = rte_mem_virt2iova(cur);
 769
 770                         iovas[cur_page] = iova;
 771                 }
 772
 773                 break;
 774         }
 775         /* if we couldn't allocate anything */
 776         if (iovas == NULL)
 777                 return -1;
 778
 779         param->addr = addr;
 780         param->len = mem_sz;
 781         param->pgsz = cur_pgsz;
 782         param->iova_table = iovas;
 783         param->iova_table_len = n_pages;
 784
 785         return 0;
 786 fail:
 787         if (iovas)
 788                 free(iovas);
 789         if (addr)
 790                 munmap(addr, mem_sz);
 791
 792         return -1;
 793 }
 794
 795 static int
 796 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 797 {
 798         struct extmem_param param;
 799         int socket_id, ret;
 800
 801         memset(&param, 0, sizeof(param));
 802
 803         /* check if our heap exists */
 804         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 805         if (socket_id < 0) {
 806                 /* create our heap */
 807                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 808                 if (ret < 0) {
 809                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 810                         return -1;
 811                 }
 812         }
 813
 814         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 815         if (ret < 0) {
 816                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 817                 return -1;
 818         }
 819
 820         /* we now have a valid memory area, so add it to heap */
 821         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 822                         param.addr, param.len, param.iova_table,
 823                         param.iova_table_len, param.pgsz);
 824
 825         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 826
 827         /* not needed any more */
 828         free(param.iova_table);
 829
 830         if (ret < 0) {
 831                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 832                 munmap(param.addr, param.len);
 833                 return -1;
 834         }
 835
 836         /* success */
 837
 838         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 839                         param.len >> 20);
 840
 841         return 0;
 842 }
 843
 844 /*
 845  * Configuration initialisation done once at init time.
 846  */
 847 static void
 848 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 849                  unsigned int socket_id)
 850 {
 851         char pool_name[RTE_MEMPOOL_NAMESIZE];
 852         struct rte_mempool *rte_mp = NULL;
 853         uint32_t mb_size;
 854
 855         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 856         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
 857
 858         TESTPMD_LOG(INFO,
 859                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 860                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 861
 862         switch (mp_alloc_type) {
 863         case MP_ALLOC_NATIVE:
 864                 {
 865                         /* wrapper to rte_mempool_create() */
 866                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 867                                         rte_mbuf_best_mempool_ops());
 868                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 869                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 870                         break;
 871                 }
 872         case MP_ALLOC_ANON:
 873                 {
 874                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 875                                 mb_size, (unsigned int) mb_mempool_cache,
 876                                 sizeof(struct rte_pktmbuf_pool_private),
 877                                 socket_id, 0);
 878                         if (rte_mp == NULL)
 879                                 goto err;
 880
 881                         if (rte_mempool_populate_anon(rte_mp) == 0) {
 882                                 rte_mempool_free(rte_mp);
 883                                 rte_mp = NULL;
 884                                 goto err;
 885                         }
 886                         rte_pktmbuf_pool_init(rte_mp, NULL);
 887                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
 888                         break;
 889                 }
 890         case MP_ALLOC_XMEM:
 891         case MP_ALLOC_XMEM_HUGE:
 892                 {
 893                         int heap_socket;
 894                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
 895
 896                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
 897                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
 898
 899                         heap_socket =
 900                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 901                         if (heap_socket < 0)
 902                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
 903
 904                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 905                                         rte_mbuf_best_mempool_ops());
 906                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 907                                         mb_mempool_cache, 0, mbuf_seg_size,
 908                                         heap_socket);
 909                         break;
 910                 }
 911         default:
 912                 {
 913                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
 914                 }
 915         }
 916
 917 err:
 918         if (rte_mp == NULL) {
 919                 rte_exit(EXIT_FAILURE,
 920                         "Creation of mbuf pool for socket %u failed: %s\n",
 921                         socket_id, rte_strerror(rte_errno));
 922         } else if (verbose_level > 0) {
 923                 rte_mempool_dump(stdout, rte_mp);
 924         }
 925 }
 926
 927 /*
 928  * Check given socket id is valid or not with NUMA mode,
 929  * if valid, return 0, else return -1
 930  */
 931 static int
 932 check_socket_id(const unsigned int socket_id)
 933 {
 934         static int warning_once = 0;
 935
 936         if (new_socket_id(socket_id)) {
 937                 if (!warning_once && numa_support)
 938                         printf("Warning: NUMA should be configured manually by"
 939                                " using --port-numa-config and"
 940                                " --ring-numa-config parameters along with"
 941                                " --numa.\n");
 942                 warning_once = 1;
 943                 return -1;
 944         }
 945         return 0;
 946 }
 947
 948 /*
 949  * Get the allowed maximum number of RX queues.
 950  * *pid return the port id which has minimal value of
 951  * max_rx_queues in all ports.
 952  */
 953 queueid_t
 954 get_allowed_max_nb_rxq(portid_t *pid)
 955 {
 956         queueid_t allowed_max_rxq = MAX_QUEUE_ID;
 957         portid_t pi;
 958         struct rte_eth_dev_info dev_info;
 959
 960         RTE_ETH_FOREACH_DEV(pi) {
 961                 rte_eth_dev_info_get(pi, &dev_info);
 962                 if (dev_info.max_rx_queues < allowed_max_rxq) {
 963                         allowed_max_rxq = dev_info.max_rx_queues;
 964                         *pid = pi;
 965                 }
 966         }
 967         return allowed_max_rxq;
 968 }
 969
 970 /*
 971  * Check input rxq is valid or not.
 972  * If input rxq is not greater than any of maximum number
 973  * of RX queues of all ports, it is valid.
 974  * if valid, return 0, else return -1
 975  */
 976 int
 977 check_nb_rxq(queueid_t rxq)
 978 {
 979         queueid_t allowed_max_rxq;
 980         portid_t pid = 0;
 981
 982         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
 983         if (rxq > allowed_max_rxq) {
 984                 printf("Fail: input rxq (%u) can't be greater "
 985                        "than max_rx_queues (%u) of port %u\n",
 986                        rxq,
 987                        allowed_max_rxq,
 988                        pid);
 989                 return -1;
 990         }
 991         return 0;
 992 }
 993
 994 /*
 995  * Get the allowed maximum number of TX queues.
 996  * *pid return the port id which has minimal value of
 997  * max_tx_queues in all ports.
 998  */
 999 queueid_t
1000 get_allowed_max_nb_txq(portid_t *pid)
1001 {
1002         queueid_t allowed_max_txq = MAX_QUEUE_ID;
1003         portid_t pi;
1004         struct rte_eth_dev_info dev_info;
1005
1006         RTE_ETH_FOREACH_DEV(pi) {
1007                 rte_eth_dev_info_get(pi, &dev_info);
1008                 if (dev_info.max_tx_queues < allowed_max_txq) {
1009                         allowed_max_txq = dev_info.max_tx_queues;
1010                         *pid = pi;
1011                 }
1012         }
1013         return allowed_max_txq;
1014 }
1015
1016 /*
1017  * Check input txq is valid or not.
1018  * If input txq is not greater than any of maximum number
1019  * of TX queues of all ports, it is valid.
1020  * if valid, return 0, else return -1
1021  */
1022 int
1023 check_nb_txq(queueid_t txq)
1024 {
1025         queueid_t allowed_max_txq;
1026         portid_t pid = 0;
1027
1028         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1029         if (txq > allowed_max_txq) {
1030                 printf("Fail: input txq (%u) can't be greater "
1031                        "than max_tx_queues (%u) of port %u\n",
1032                        txq,
1033                        allowed_max_txq,
1034                        pid);
1035                 return -1;
1036         }
1037         return 0;
1038 }
1039
1040 static void
1041 init_config(void)
1042 {
1043         portid_t pid;
1044         struct rte_port *port;
1045         struct rte_mempool *mbp;
1046         unsigned int nb_mbuf_per_pool;
1047         lcoreid_t  lc_id;
1048         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1049         struct rte_gro_param gro_param;
1050         uint32_t gso_types;
1051         int k;
1052
1053         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1054
1055         /* Configuration of logical cores. */
1056         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1057                                 sizeof(struct fwd_lcore *) * nb_lcores,
1058                                 RTE_CACHE_LINE_SIZE);
1059         if (fwd_lcores == NULL) {
1060                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1061                                                         "failed\n", nb_lcores);
1062         }
1063         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1064                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1065                                                sizeof(struct fwd_lcore),
1066                                                RTE_CACHE_LINE_SIZE);
1067                 if (fwd_lcores[lc_id] == NULL) {
1068                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1069                                                                 "failed\n");
1070                 }
1071                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1072         }
1073
1074         RTE_ETH_FOREACH_DEV(pid) {
1075                 port = &ports[pid];
1076                 /* Apply default TxRx configuration for all ports */
1077                 port->dev_conf.txmode = tx_mode;
1078                 port->dev_conf.rxmode = rx_mode;
1079                 rte_eth_dev_info_get(pid, &port->dev_info);
1080
1081                 if (!(port->dev_info.tx_offload_capa &
1082                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1083                         port->dev_conf.txmode.offloads &=
1084                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1085                 if (!(port->dev_info.tx_offload_capa &
1086                         DEV_TX_OFFLOAD_MATCH_METADATA))
1087                         port->dev_conf.txmode.offloads &=
1088                                 ~DEV_TX_OFFLOAD_MATCH_METADATA;
1089                 if (numa_support) {
1090                         if (port_numa[pid] != NUMA_NO_CONFIG)
1091                                 port_per_socket[port_numa[pid]]++;
1092                         else {
1093                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1094
1095                                 /*
1096                                  * if socket_id is invalid,
1097                                  * set to the first available socket.
1098                                  */
1099                                 if (check_socket_id(socket_id) < 0)
1100                                         socket_id = socket_ids[0];
1101                                 port_per_socket[socket_id]++;
1102                         }
1103                 }
1104
1105                 /* Apply Rx offloads configuration */
1106                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1107                         port->rx_conf[k].offloads =
1108                                 port->dev_conf.rxmode.offloads;
1109                 /* Apply Tx offloads configuration */
1110                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1111                         port->tx_conf[k].offloads =
1112                                 port->dev_conf.txmode.offloads;
1113
1114                 /* set flag to initialize port/queue */
1115                 port->need_reconfig = 1;
1116                 port->need_reconfig_queues = 1;
1117                 port->tx_metadata = 0;
1118         }
1119
1120         /*
1121          * Create pools of mbuf.
1122          * If NUMA support is disabled, create a single pool of mbuf in
1123          * socket 0 memory by default.
1124          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1125          *
1126          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1127          * nb_txd can be configured at run time.
1128          */
1129         if (param_total_num_mbufs)
1130                 nb_mbuf_per_pool = param_total_num_mbufs;
1131         else {
1132                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1133                         (nb_lcores * mb_mempool_cache) +
1134                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1135                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1136         }
1137
1138         if (numa_support) {
1139                 uint8_t i;
1140
1141                 for (i = 0; i < num_sockets; i++)
1142                         mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool,
1143                                          socket_ids[i]);
1144         } else {
1145                 if (socket_num == UMA_NO_CONFIG)
1146                         mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 0);
1147                 else
1148                         mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool,
1149                                                  socket_num);
1150         }
1151
1152         init_port_config();
1153
1154         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1155                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1156         /*
1157          * Records which Mbuf pool to use by each logical core, if needed.
1158          */
1159         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1160                 mbp = mbuf_pool_find(
1161                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]));
1162
1163                 if (mbp == NULL)
1164                         mbp = mbuf_pool_find(0);
1165                 fwd_lcores[lc_id]->mbp = mbp;
1166                 /* initialize GSO context */
1167                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1168                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1169                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1170                 fwd_lcores[lc_id]->gso_ctx.gso_size = ETHER_MAX_LEN -
1171                         ETHER_CRC_LEN;
1172                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1173         }
1174
1175         /* Configuration of packet forwarding streams. */
1176         if (init_fwd_streams() < 0)
1177                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1178
1179         fwd_config_setup();
1180
1181         /* create a gro context for each lcore */
1182         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1183         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1184         gro_param.max_item_per_flow = MAX_PKT_BURST;
1185         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1186                 gro_param.socket_id = rte_lcore_to_socket_id(
1187                                 fwd_lcores_cpuids[lc_id]);
1188                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1189                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1190                         rte_exit(EXIT_FAILURE,
1191                                         "rte_gro_ctx_create() failed\n");
1192                 }
1193         }
1194
1195 #if defined RTE_LIBRTE_PMD_SOFTNIC
1196         if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) {
1197                 RTE_ETH_FOREACH_DEV(pid) {
1198                         port = &ports[pid];
1199                         const char *driver = port->dev_info.driver_name;
1200
1201                         if (strcmp(driver, "net_softnic") == 0)
1202                                 port->softport.fwd_lcore_arg = fwd_lcores;
1203                 }
1204         }
1205 #endif
1206
1207 }
1208
1209
1210 void
1211 reconfig(portid_t new_port_id, unsigned socket_id)
1212 {
1213         struct rte_port *port;
1214
1215         /* Reconfiguration of Ethernet ports. */
1216         port = &ports[new_port_id];
1217         rte_eth_dev_info_get(new_port_id, &port->dev_info);
1218
1219         /* set flag to initialize port/queue */
1220         port->need_reconfig = 1;
1221         port->need_reconfig_queues = 1;
1222         port->socket_id = socket_id;
1223
1224         init_port_config();
1225 }
1226
1227
1228 int
1229 init_fwd_streams(void)
1230 {
1231         portid_t pid;
1232         struct rte_port *port;
1233         streamid_t sm_id, nb_fwd_streams_new;
1234         queueid_t q;
1235
1236         /* set socket id according to numa or not */
1237         RTE_ETH_FOREACH_DEV(pid) {
1238                 port = &ports[pid];
1239                 if (nb_rxq > port->dev_info.max_rx_queues) {
1240                         printf("Fail: nb_rxq(%d) is greater than "
1241                                 "max_rx_queues(%d)\n", nb_rxq,
1242                                 port->dev_info.max_rx_queues);
1243                         return -1;
1244                 }
1245                 if (nb_txq > port->dev_info.max_tx_queues) {
1246                         printf("Fail: nb_txq(%d) is greater than "
1247                                 "max_tx_queues(%d)\n", nb_txq,
1248                                 port->dev_info.max_tx_queues);
1249                         return -1;
1250                 }
1251                 if (numa_support) {
1252                         if (port_numa[pid] != NUMA_NO_CONFIG)
1253                                 port->socket_id = port_numa[pid];
1254                         else {
1255                                 port->socket_id = rte_eth_dev_socket_id(pid);
1256
1257                                 /*
1258                                  * if socket_id is invalid,
1259                                  * set to the first available socket.
1260                                  */
1261                                 if (check_socket_id(port->socket_id) < 0)
1262                                         port->socket_id = socket_ids[0];
1263                         }
1264                 }
1265                 else {
1266                         if (socket_num == UMA_NO_CONFIG)
1267                                 port->socket_id = 0;
1268                         else
1269                                 port->socket_id = socket_num;
1270                 }
1271         }
1272
1273         q = RTE_MAX(nb_rxq, nb_txq);
1274         if (q == 0) {
1275                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1276                 return -1;
1277         }
1278         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1279         if (nb_fwd_streams_new == nb_fwd_streams)
1280                 return 0;
1281         /* clear the old */
1282         if (fwd_streams != NULL) {
1283                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1284                         if (fwd_streams[sm_id] == NULL)
1285                                 continue;
1286                         rte_free(fwd_streams[sm_id]);
1287                         fwd_streams[sm_id] = NULL;
1288                 }
1289                 rte_free(fwd_streams);
1290                 fwd_streams = NULL;
1291         }
1292
1293         /* init new */
1294         nb_fwd_streams = nb_fwd_streams_new;
1295         if (nb_fwd_streams) {
1296                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1297                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1298                         RTE_CACHE_LINE_SIZE);
1299                 if (fwd_streams == NULL)
1300                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1301                                  " (struct fwd_stream *)) failed\n",
1302                                  nb_fwd_streams);
1303
1304                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1305                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1306                                 " struct fwd_stream", sizeof(struct fwd_stream),
1307                                 RTE_CACHE_LINE_SIZE);
1308                         if (fwd_streams[sm_id] == NULL)
1309                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1310                                          "(struct fwd_stream) failed\n");
1311                 }
1312         }
1313
1314         return 0;
1315 }
1316
1317 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1318 static void
1319 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1320 {
1321         unsigned int total_burst;
1322         unsigned int nb_burst;
1323         unsigned int burst_stats[3];
1324         uint16_t pktnb_stats[3];
1325         uint16_t nb_pkt;
1326         int burst_percent[3];
1327
1328         /*
1329          * First compute the total number of packet bursts and the
1330          * two highest numbers of bursts of the same number of packets.
1331          */
1332         total_burst = 0;
1333         burst_stats[0] = burst_stats[1] = burst_stats[2] = 0;
1334         pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0;
1335         for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1336                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1337                 if (nb_burst == 0)
1338                         continue;
1339                 total_burst += nb_burst;
1340                 if (nb_burst > burst_stats[0]) {
1341                         burst_stats[1] = burst_stats[0];
1342                         pktnb_stats[1] = pktnb_stats[0];
1343                         burst_stats[0] = nb_burst;
1344                         pktnb_stats[0] = nb_pkt;
1345                 } else if (nb_burst > burst_stats[1]) {
1346                         burst_stats[1] = nb_burst;
1347                         pktnb_stats[1] = nb_pkt;
1348                 }
1349         }
1350         if (total_burst == 0)
1351                 return;
1352         burst_percent[0] = (burst_stats[0] * 100) / total_burst;
1353         printf("  %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst,
1354                burst_percent[0], (int) pktnb_stats[0]);
1355         if (burst_stats[0] == total_burst) {
1356                 printf("]\n");
1357                 return;
1358         }
1359         if (burst_stats[0] + burst_stats[1] == total_burst) {
1360                 printf(" + %d%% of %d pkts]\n",
1361                        100 - burst_percent[0], pktnb_stats[1]);
1362                 return;
1363         }
1364         burst_percent[1] = (burst_stats[1] * 100) / total_burst;
1365         burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]);
1366         if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) {
1367                 printf(" + %d%% of others]\n", 100 - burst_percent[0]);
1368                 return;
1369         }
1370         printf(" + %d%% of %d pkts + %d%% of others]\n",
1371                burst_percent[1], (int) pktnb_stats[1], burst_percent[2]);
1372 }
1373 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
1374
1375 static void
1376 fwd_port_stats_display(portid_t port_id, struct rte_eth_stats *stats)
1377 {
1378         struct rte_port *port;
1379         uint8_t i;
1380
1381         static const char *fwd_stats_border = "----------------------";
1382
1383         port = &ports[port_id];
1384         printf("\n  %s Forward statistics for port %-2d %s\n",
1385                fwd_stats_border, port_id, fwd_stats_border);
1386
1387         if ((!port->rx_queue_stats_mapping_enabled) && (!port->tx_queue_stats_mapping_enabled)) {
1388                 printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1389                        "%-"PRIu64"\n",
1390                        stats->ipackets, stats->imissed,
1391                        (uint64_t) (stats->ipackets + stats->imissed));
1392
1393                 if (cur_fwd_eng == &csum_fwd_engine)
1394                         printf("  Bad-ipcsum: %-14"PRIu64" Bad-l4csum: %-14"PRIu64"Bad-outer-l4csum: %-14"PRIu64"\n",
1395                                port->rx_bad_ip_csum, port->rx_bad_l4_csum,
1396                                port->rx_bad_outer_l4_csum);
1397                 if ((stats->ierrors + stats->rx_nombuf) > 0) {
1398                         printf("  RX-error: %-"PRIu64"\n",  stats->ierrors);
1399                         printf("  RX-nombufs: %-14"PRIu64"\n", stats->rx_nombuf);
1400                 }
1401
1402                 printf("  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1403                        "%-"PRIu64"\n",
1404                        stats->opackets, port->tx_dropped,
1405                        (uint64_t) (stats->opackets + port->tx_dropped));
1406         }
1407         else {
1408                 printf("  RX-packets:             %14"PRIu64"    RX-dropped:%14"PRIu64"    RX-total:"
1409                        "%14"PRIu64"\n",
1410                        stats->ipackets, stats->imissed,
1411                        (uint64_t) (stats->ipackets + stats->imissed));
1412
1413                 if (cur_fwd_eng == &csum_fwd_engine)
1414                         printf("  Bad-ipcsum:%14"PRIu64"    Bad-l4csum:%14"PRIu64"    Bad-outer-l4csum: %-14"PRIu64"\n",
1415                                port->rx_bad_ip_csum, port->rx_bad_l4_csum,
1416                                port->rx_bad_outer_l4_csum);
1417                 if ((stats->ierrors + stats->rx_nombuf) > 0) {
1418                         printf("  RX-error:%"PRIu64"\n", stats->ierrors);
1419                         printf("  RX-nombufs:             %14"PRIu64"\n",
1420                                stats->rx_nombuf);
1421                 }
1422
1423                 printf("  TX-packets:             %14"PRIu64"    TX-dropped:%14"PRIu64"    TX-total:"
1424                        "%14"PRIu64"\n",
1425                        stats->opackets, port->tx_dropped,
1426                        (uint64_t) (stats->opackets + port->tx_dropped));
1427         }
1428
1429 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1430         if (port->rx_stream)
1431                 pkt_burst_stats_display("RX",
1432                         &port->rx_stream->rx_burst_stats);
1433         if (port->tx_stream)
1434                 pkt_burst_stats_display("TX",
1435                         &port->tx_stream->tx_burst_stats);
1436 #endif
1437
1438         if (port->rx_queue_stats_mapping_enabled) {
1439                 printf("\n");
1440                 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) {
1441                         printf("  Stats reg %2d RX-packets:%14"PRIu64
1442                                "     RX-errors:%14"PRIu64
1443                                "    RX-bytes:%14"PRIu64"\n",
1444                                i, stats->q_ipackets[i], stats->q_errors[i], stats->q_ibytes[i]);
1445                 }
1446                 printf("\n");
1447         }
1448         if (port->tx_queue_stats_mapping_enabled) {
1449                 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) {
1450                         printf("  Stats reg %2d TX-packets:%14"PRIu64
1451                                "                                 TX-bytes:%14"PRIu64"\n",
1452                                i, stats->q_opackets[i], stats->q_obytes[i]);
1453                 }
1454         }
1455
1456         printf("  %s--------------------------------%s\n",
1457                fwd_stats_border, fwd_stats_border);
1458 }
1459
1460 static void
1461 fwd_stream_stats_display(streamid_t stream_id)
1462 {
1463         struct fwd_stream *fs;
1464         static const char *fwd_top_stats_border = "-------";
1465
1466         fs = fwd_streams[stream_id];
1467         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1468             (fs->fwd_dropped == 0))
1469                 return;
1470         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1471                "TX Port=%2d/Queue=%2d %s\n",
1472                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1473                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1474         printf("  RX-packets: %-14u TX-packets: %-14u TX-dropped: %-14u",
1475                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1476
1477         /* if checksum mode */
1478         if (cur_fwd_eng == &csum_fwd_engine) {
1479                printf("  RX- bad IP checksum: %-14u  Rx- bad L4 checksum: "
1480                         "%-14u Rx- bad outer L4 checksum: %-14u\n",
1481                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1482                         fs->rx_bad_outer_l4_csum);
1483         }
1484
1485 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1486         pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1487         pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1488 #endif
1489 }
1490
1491 static void
1492 flush_fwd_rx_queues(void)
1493 {
1494         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1495         portid_t  rxp;
1496         portid_t port_id;
1497         queueid_t rxq;
1498         uint16_t  nb_rx;
1499         uint16_t  i;
1500         uint8_t   j;
1501         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
1502         uint64_t timer_period;
1503
1504         /* convert to number of cycles */
1505         timer_period = rte_get_timer_hz(); /* 1 second timeout */
1506
1507         for (j = 0; j < 2; j++) {
1508                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
1509                         for (rxq = 0; rxq < nb_rxq; rxq++) {
1510                                 port_id = fwd_ports_ids[rxp];
1511                                 /**
1512                                 * testpmd can stuck in the below do while loop
1513                                 * if rte_eth_rx_burst() always returns nonzero
1514                                 * packets. So timer is added to exit this loop
1515                                 * after 1sec timer expiry.
1516                                 */
1517                                 prev_tsc = rte_rdtsc();
1518                                 do {
1519                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
1520                                                 pkts_burst, MAX_PKT_BURST);
1521                                         for (i = 0; i < nb_rx; i++)
1522                                                 rte_pktmbuf_free(pkts_burst[i]);
1523
1524                                         cur_tsc = rte_rdtsc();
1525                                         diff_tsc = cur_tsc - prev_tsc;
1526                                         timer_tsc += diff_tsc;
1527                                 } while ((nb_rx > 0) &&
1528                                         (timer_tsc < timer_period));
1529                                 timer_tsc = 0;
1530                         }
1531                 }
1532                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
1533         }
1534 }
1535
1536 static void
1537 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
1538 {
1539         struct fwd_stream **fsm;
1540         streamid_t nb_fs;
1541         streamid_t sm_id;
1542 #ifdef RTE_LIBRTE_BITRATE
1543         uint64_t tics_per_1sec;
1544         uint64_t tics_datum;
1545         uint64_t tics_current;
1546         uint16_t i, cnt_ports;
1547
1548         cnt_ports = nb_ports;
1549         tics_datum = rte_rdtsc();
1550         tics_per_1sec = rte_get_timer_hz();
1551 #endif
1552         fsm = &fwd_streams[fc->stream_idx];
1553         nb_fs = fc->stream_nb;
1554         do {
1555                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
1556                         (*pkt_fwd)(fsm[sm_id]);
1557 #ifdef RTE_LIBRTE_BITRATE
1558                 if (bitrate_enabled != 0 &&
1559                                 bitrate_lcore_id == rte_lcore_id()) {
1560                         tics_current = rte_rdtsc();
1561                         if (tics_current - tics_datum >= tics_per_1sec) {
1562                                 /* Periodic bitrate calculation */
1563                                 for (i = 0; i < cnt_ports; i++)
1564                                         rte_stats_bitrate_calc(bitrate_data,
1565                                                 ports_ids[i]);
1566                                 tics_datum = tics_current;
1567                         }
1568                 }
1569 #endif
1570 #ifdef RTE_LIBRTE_LATENCY_STATS
1571                 if (latencystats_enabled != 0 &&
1572                                 latencystats_lcore_id == rte_lcore_id())
1573                         rte_latencystats_update();
1574 #endif
1575
1576         } while (! fc->stopped);
1577 }
1578
1579 static int
1580 start_pkt_forward_on_core(void *fwd_arg)
1581 {
1582         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
1583                              cur_fwd_config.fwd_eng->packet_fwd);
1584         return 0;
1585 }
1586
1587 /*
1588  * Run the TXONLY packet forwarding engine to send a single burst of packets.
1589  * Used to start communication flows in network loopback test configurations.
1590  */
1591 static int
1592 run_one_txonly_burst_on_core(void *fwd_arg)
1593 {
1594         struct fwd_lcore *fwd_lc;
1595         struct fwd_lcore tmp_lcore;
1596
1597         fwd_lc = (struct fwd_lcore *) fwd_arg;
1598         tmp_lcore = *fwd_lc;
1599         tmp_lcore.stopped = 1;
1600         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
1601         return 0;
1602 }
1603
1604 /*
1605  * Launch packet forwarding:
1606  *     - Setup per-port forwarding context.
1607  *     - launch logical cores with their forwarding configuration.
1608  */
1609 static void
1610 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
1611 {
1612         port_fwd_begin_t port_fwd_begin;
1613         unsigned int i;
1614         unsigned int lc_id;
1615         int diag;
1616
1617         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
1618         if (port_fwd_begin != NULL) {
1619                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1620                         (*port_fwd_begin)(fwd_ports_ids[i]);
1621         }
1622         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
1623                 lc_id = fwd_lcores_cpuids[i];
1624                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
1625                         fwd_lcores[i]->stopped = 0;
1626                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
1627                                                      fwd_lcores[i], lc_id);
1628                         if (diag != 0)
1629                                 printf("launch lcore %u failed - diag=%d\n",
1630                                        lc_id, diag);
1631                 }
1632         }
1633 }
1634
1635 /*
1636  * Launch packet forwarding configuration.
1637  */
1638 void
1639 start_packet_forwarding(int with_tx_first)
1640 {
1641         port_fwd_begin_t port_fwd_begin;
1642         port_fwd_end_t  port_fwd_end;
1643         struct rte_port *port;
1644         unsigned int i;
1645         portid_t   pt_id;
1646         streamid_t sm_id;
1647
1648         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
1649                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
1650
1651         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
1652                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
1653
1654         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
1655                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
1656                 (!nb_rxq || !nb_txq))
1657                 rte_exit(EXIT_FAILURE,
1658                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
1659                         cur_fwd_eng->fwd_mode_name);
1660
1661         if (all_ports_started() == 0) {
1662                 printf("Not all ports were started\n");
1663                 return;
1664         }
1665         if (test_done == 0) {
1666                 printf("Packet forwarding already started\n");
1667                 return;
1668         }
1669
1670
1671         if(dcb_test) {
1672                 for (i = 0; i < nb_fwd_ports; i++) {
1673                         pt_id = fwd_ports_ids[i];
1674                         port = &ports[pt_id];
1675                         if (!port->dcb_flag) {
1676                                 printf("In DCB mode, all forwarding ports must "
1677                                        "be configured in this mode.\n");
1678                                 return;
1679                         }
1680                 }
1681                 if (nb_fwd_lcores == 1) {
1682                         printf("In DCB mode,the nb forwarding cores "
1683                                "should be larger than 1.\n");
1684                         return;
1685                 }
1686         }
1687         test_done = 0;
1688
1689         fwd_config_setup();
1690
1691         if(!no_flush_rx)
1692                 flush_fwd_rx_queues();
1693
1694         pkt_fwd_config_display(&cur_fwd_config);
1695         rxtx_config_display();
1696
1697         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1698                 pt_id = fwd_ports_ids[i];
1699                 port = &ports[pt_id];
1700                 rte_eth_stats_get(pt_id, &port->stats);
1701                 port->tx_dropped = 0;
1702
1703                 map_port_queue_stats_mapping_registers(pt_id, port);
1704         }
1705         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1706                 fwd_streams[sm_id]->rx_packets = 0;
1707                 fwd_streams[sm_id]->tx_packets = 0;
1708                 fwd_streams[sm_id]->fwd_dropped = 0;
1709                 fwd_streams[sm_id]->rx_bad_ip_csum = 0;
1710                 fwd_streams[sm_id]->rx_bad_l4_csum = 0;
1711                 fwd_streams[sm_id]->rx_bad_outer_l4_csum = 0;
1712
1713 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1714                 memset(&fwd_streams[sm_id]->rx_burst_stats, 0,
1715                        sizeof(fwd_streams[sm_id]->rx_burst_stats));
1716                 memset(&fwd_streams[sm_id]->tx_burst_stats, 0,
1717                        sizeof(fwd_streams[sm_id]->tx_burst_stats));
1718 #endif
1719 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1720                 fwd_streams[sm_id]->core_cycles = 0;
1721 #endif
1722         }
1723         if (with_tx_first) {
1724                 port_fwd_begin = tx_only_engine.port_fwd_begin;
1725                 if (port_fwd_begin != NULL) {
1726                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1727                                 (*port_fwd_begin)(fwd_ports_ids[i]);
1728                 }
1729                 while (with_tx_first--) {
1730                         launch_packet_forwarding(
1731                                         run_one_txonly_burst_on_core);
1732                         rte_eal_mp_wait_lcore();
1733                 }
1734                 port_fwd_end = tx_only_engine.port_fwd_end;
1735                 if (port_fwd_end != NULL) {
1736                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1737                                 (*port_fwd_end)(fwd_ports_ids[i]);
1738                 }
1739         }
1740         launch_packet_forwarding(start_pkt_forward_on_core);
1741 }
1742
1743 void
1744 stop_packet_forwarding(void)
1745 {
1746         struct rte_eth_stats stats;
1747         struct rte_port *port;
1748         port_fwd_end_t  port_fwd_end;
1749         int i;
1750         portid_t   pt_id;
1751         streamid_t sm_id;
1752         lcoreid_t  lc_id;
1753         uint64_t total_recv;
1754         uint64_t total_xmit;
1755         uint64_t total_rx_dropped;
1756         uint64_t total_tx_dropped;
1757         uint64_t total_rx_nombuf;
1758         uint64_t tx_dropped;
1759         uint64_t rx_bad_ip_csum;
1760         uint64_t rx_bad_l4_csum;
1761 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1762         uint64_t fwd_cycles;
1763 #endif
1764
1765         static const char *acc_stats_border = "+++++++++++++++";
1766
1767         if (test_done) {
1768                 printf("Packet forwarding not started\n");
1769                 return;
1770         }
1771         printf("Telling cores to stop...");
1772         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
1773                 fwd_lcores[lc_id]->stopped = 1;
1774         printf("\nWaiting for lcores to finish...\n");
1775         rte_eal_mp_wait_lcore();
1776         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
1777         if (port_fwd_end != NULL) {
1778                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1779                         pt_id = fwd_ports_ids[i];
1780                         (*port_fwd_end)(pt_id);
1781                 }
1782         }
1783 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1784         fwd_cycles = 0;
1785 #endif
1786         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1787                 if (cur_fwd_config.nb_fwd_streams >
1788                     cur_fwd_config.nb_fwd_ports) {
1789                         fwd_stream_stats_display(sm_id);
1790                         ports[fwd_streams[sm_id]->tx_port].tx_stream = NULL;
1791                         ports[fwd_streams[sm_id]->rx_port].rx_stream = NULL;
1792                 } else {
1793                         ports[fwd_streams[sm_id]->tx_port].tx_stream =
1794                                 fwd_streams[sm_id];
1795                         ports[fwd_streams[sm_id]->rx_port].rx_stream =
1796                                 fwd_streams[sm_id];
1797                 }
1798                 tx_dropped = ports[fwd_streams[sm_id]->tx_port].tx_dropped;
1799                 tx_dropped = (uint64_t) (tx_dropped +
1800                                          fwd_streams[sm_id]->fwd_dropped);
1801                 ports[fwd_streams[sm_id]->tx_port].tx_dropped = tx_dropped;
1802
1803                 rx_bad_ip_csum =
1804                         ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum;
1805                 rx_bad_ip_csum = (uint64_t) (rx_bad_ip_csum +
1806                                          fwd_streams[sm_id]->rx_bad_ip_csum);
1807                 ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum =
1808                                                         rx_bad_ip_csum;
1809
1810                 rx_bad_l4_csum =
1811                         ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum;
1812                 rx_bad_l4_csum = (uint64_t) (rx_bad_l4_csum +
1813                                          fwd_streams[sm_id]->rx_bad_l4_csum);
1814                 ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum =
1815                                                         rx_bad_l4_csum;
1816
1817                 ports[fwd_streams[sm_id]->rx_port].rx_bad_outer_l4_csum +=
1818                                 fwd_streams[sm_id]->rx_bad_outer_l4_csum;
1819
1820 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1821                 fwd_cycles = (uint64_t) (fwd_cycles +
1822                                          fwd_streams[sm_id]->core_cycles);
1823 #endif
1824         }
1825         total_recv = 0;
1826         total_xmit = 0;
1827         total_rx_dropped = 0;
1828         total_tx_dropped = 0;
1829         total_rx_nombuf  = 0;
1830         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1831                 pt_id = fwd_ports_ids[i];
1832
1833                 port = &ports[pt_id];
1834                 rte_eth_stats_get(pt_id, &stats);
1835                 stats.ipackets -= port->stats.ipackets;
1836                 port->stats.ipackets = 0;
1837                 stats.opackets -= port->stats.opackets;
1838                 port->stats.opackets = 0;
1839                 stats.ibytes   -= port->stats.ibytes;
1840                 port->stats.ibytes = 0;
1841                 stats.obytes   -= port->stats.obytes;
1842                 port->stats.obytes = 0;
1843                 stats.imissed  -= port->stats.imissed;
1844                 port->stats.imissed = 0;
1845                 stats.oerrors  -= port->stats.oerrors;
1846                 port->stats.oerrors = 0;
1847                 stats.rx_nombuf -= port->stats.rx_nombuf;
1848                 port->stats.rx_nombuf = 0;
1849
1850                 total_recv += stats.ipackets;
1851                 total_xmit += stats.opackets;
1852                 total_rx_dropped += stats.imissed;
1853                 total_tx_dropped += port->tx_dropped;
1854                 total_rx_nombuf  += stats.rx_nombuf;
1855
1856                 fwd_port_stats_display(pt_id, &stats);
1857         }
1858
1859         printf("\n  %s Accumulated forward statistics for all ports"
1860                "%s\n",
1861                acc_stats_border, acc_stats_border);
1862         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1863                "%-"PRIu64"\n"
1864                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1865                "%-"PRIu64"\n",
1866                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1867                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1868         if (total_rx_nombuf > 0)
1869                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1870         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1871                "%s\n",
1872                acc_stats_border, acc_stats_border);
1873 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1874         if (total_recv > 0)
1875                 printf("\n  CPU cycles/packet=%u (total cycles="
1876                        "%"PRIu64" / total RX packets=%"PRIu64")\n",
1877                        (unsigned int)(fwd_cycles / total_recv),
1878                        fwd_cycles, total_recv);
1879 #endif
1880         printf("\nDone.\n");
1881         test_done = 1;
1882 }
1883
1884 void
1885 dev_set_link_up(portid_t pid)
1886 {
1887         if (rte_eth_dev_set_link_up(pid) < 0)
1888                 printf("\nSet link up fail.\n");
1889 }
1890
1891 void
1892 dev_set_link_down(portid_t pid)
1893 {
1894         if (rte_eth_dev_set_link_down(pid) < 0)
1895                 printf("\nSet link down fail.\n");
1896 }
1897
1898 static int
1899 all_ports_started(void)
1900 {
1901         portid_t pi;
1902         struct rte_port *port;
1903
1904         RTE_ETH_FOREACH_DEV(pi) {
1905                 port = &ports[pi];
1906                 /* Check if there is a port which is not started */
1907                 if ((port->port_status != RTE_PORT_STARTED) &&
1908                         (port->slave_flag == 0))
1909                         return 0;
1910         }
1911
1912         /* No port is not started */
1913         return 1;
1914 }
1915
1916 int
1917 port_is_stopped(portid_t port_id)
1918 {
1919         struct rte_port *port = &ports[port_id];
1920
1921         if ((port->port_status != RTE_PORT_STOPPED) &&
1922             (port->slave_flag == 0))
1923                 return 0;
1924         return 1;
1925 }
1926
1927 int
1928 all_ports_stopped(void)
1929 {
1930         portid_t pi;
1931
1932         RTE_ETH_FOREACH_DEV(pi) {
1933                 if (!port_is_stopped(pi))
1934                         return 0;
1935         }
1936
1937         return 1;
1938 }
1939
1940 int
1941 port_is_started(portid_t port_id)
1942 {
1943         if (port_id_is_invalid(port_id, ENABLED_WARN))
1944                 return 0;
1945
1946         if (ports[port_id].port_status != RTE_PORT_STARTED)
1947                 return 0;
1948
1949         return 1;
1950 }
1951
1952 int
1953 start_port(portid_t pid)
1954 {
1955         int diag, need_check_link_status = -1;
1956         portid_t pi;
1957         queueid_t qi;
1958         struct rte_port *port;
1959         struct ether_addr mac_addr;
1960
1961         if (port_id_is_invalid(pid, ENABLED_WARN))
1962                 return 0;
1963
1964         if(dcb_config)
1965                 dcb_test = 1;
1966         RTE_ETH_FOREACH_DEV(pi) {
1967                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
1968                         continue;
1969
1970                 need_check_link_status = 0;
1971                 port = &ports[pi];
1972                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
1973                                                  RTE_PORT_HANDLING) == 0) {
1974                         printf("Port %d is now not stopped\n", pi);
1975                         continue;
1976                 }
1977
1978                 if (port->need_reconfig > 0) {
1979                         port->need_reconfig = 0;
1980
1981                         if (flow_isolate_all) {
1982                                 int ret = port_flow_isolate(pi, 1);
1983                                 if (ret) {
1984                                         printf("Failed to apply isolated"
1985                                                " mode on port %d\n", pi);
1986                                         return -1;
1987                                 }
1988                         }
1989                         configure_rxtx_dump_callbacks(0);
1990                         printf("Configuring Port %d (socket %u)\n", pi,
1991                                         port->socket_id);
1992                         /* configure port */
1993                         diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq,
1994                                                 &(port->dev_conf));
1995                         if (diag != 0) {
1996                                 if (rte_atomic16_cmpset(&(port->port_status),
1997                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
1998                                         printf("Port %d can not be set back "
1999                                                         "to stopped\n", pi);
2000                                 printf("Fail to configure port %d\n", pi);
2001                                 /* try to reconfigure port next time */
2002                                 port->need_reconfig = 1;
2003                                 return -1;
2004                         }
2005                 }
2006                 if (port->need_reconfig_queues > 0) {
2007                         port->need_reconfig_queues = 0;
2008                         /* setup tx queues */
2009                         for (qi = 0; qi < nb_txq; qi++) {
2010                                 if ((numa_support) &&
2011                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2012                                         diag = rte_eth_tx_queue_setup(pi, qi,
2013                                                 port->nb_tx_desc[qi],
2014                                                 txring_numa[pi],
2015                                                 &(port->tx_conf[qi]));
2016                                 else
2017                                         diag = rte_eth_tx_queue_setup(pi, qi,
2018                                                 port->nb_tx_desc[qi],
2019                                                 port->socket_id,
2020                                                 &(port->tx_conf[qi]));
2021
2022                                 if (diag == 0)
2023                                         continue;
2024
2025                                 /* Fail to setup tx queue, return */
2026                                 if (rte_atomic16_cmpset(&(port->port_status),
2027                                                         RTE_PORT_HANDLING,
2028                                                         RTE_PORT_STOPPED) == 0)
2029                                         printf("Port %d can not be set back "
2030                                                         "to stopped\n", pi);
2031                                 printf("Fail to configure port %d tx queues\n",
2032                                        pi);
2033                                 /* try to reconfigure queues next time */
2034                                 port->need_reconfig_queues = 1;
2035                                 return -1;
2036                         }
2037                         for (qi = 0; qi < nb_rxq; qi++) {
2038                                 /* setup rx queues */
2039                                 if ((numa_support) &&
2040                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2041                                         struct rte_mempool * mp =
2042                                                 mbuf_pool_find(rxring_numa[pi]);
2043                                         if (mp == NULL) {
2044                                                 printf("Failed to setup RX queue:"
2045                                                         "No mempool allocation"
2046                                                         " on the socket %d\n",
2047                                                         rxring_numa[pi]);
2048                                                 return -1;
2049                                         }
2050
2051                                         diag = rte_eth_rx_queue_setup(pi, qi,
2052                                              port->nb_rx_desc[qi],
2053                                              rxring_numa[pi],
2054                                              &(port->rx_conf[qi]),
2055                                              mp);
2056                                 } else {
2057                                         struct rte_mempool *mp =
2058                                                 mbuf_pool_find(port->socket_id);
2059                                         if (mp == NULL) {
2060                                                 printf("Failed to setup RX queue:"
2061                                                         "No mempool allocation"
2062                                                         " on the socket %d\n",
2063                                                         port->socket_id);
2064                                                 return -1;
2065                                         }
2066                                         diag = rte_eth_rx_queue_setup(pi, qi,
2067                                              port->nb_rx_desc[qi],
2068                                              port->socket_id,
2069                                              &(port->rx_conf[qi]),
2070                                              mp);
2071                                 }
2072                                 if (diag == 0)
2073                                         continue;
2074
2075                                 /* Fail to setup rx queue, return */
2076                                 if (rte_atomic16_cmpset(&(port->port_status),
2077                                                         RTE_PORT_HANDLING,
2078                                                         RTE_PORT_STOPPED) == 0)
2079                                         printf("Port %d can not be set back "
2080                                                         "to stopped\n", pi);
2081                                 printf("Fail to configure port %d rx queues\n",
2082                                        pi);
2083                                 /* try to reconfigure queues next time */
2084                                 port->need_reconfig_queues = 1;
2085                                 return -1;
2086                         }
2087                 }
2088                 configure_rxtx_dump_callbacks(verbose_level);
2089                 /* start port */
2090                 if (rte_eth_dev_start(pi) < 0) {
2091                         printf("Fail to start port %d\n", pi);
2092
2093                         /* Fail to setup rx queue, return */
2094                         if (rte_atomic16_cmpset(&(port->port_status),
2095                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2096                                 printf("Port %d can not be set back to "
2097                                                         "stopped\n", pi);
2098                         continue;
2099                 }
2100
2101                 if (rte_atomic16_cmpset(&(port->port_status),
2102                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2103                         printf("Port %d can not be set into started\n", pi);
2104
2105                 rte_eth_macaddr_get(pi, &mac_addr);
2106                 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2107                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2108                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2109                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2110
2111                 /* at least one port started, need checking link status */
2112                 need_check_link_status = 1;
2113         }
2114
2115         if (need_check_link_status == 1 && !no_link_check)
2116                 check_all_ports_link_status(RTE_PORT_ALL);
2117         else if (need_check_link_status == 0)
2118                 printf("Please stop the ports first\n");
2119
2120         printf("Done\n");
2121         return 0;
2122 }
2123
2124 void
2125 stop_port(portid_t pid)
2126 {
2127         portid_t pi;
2128         struct rte_port *port;
2129         int need_check_link_status = 0;
2130
2131         if (dcb_test) {
2132                 dcb_test = 0;
2133                 dcb_config = 0;
2134         }
2135
2136         if (port_id_is_invalid(pid, ENABLED_WARN))
2137                 return;
2138
2139         printf("Stopping ports...\n");
2140
2141         RTE_ETH_FOREACH_DEV(pi) {
2142                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2143                         continue;
2144
2145                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2146                         printf("Please remove port %d from forwarding configuration.\n", pi);
2147                         continue;
2148                 }
2149
2150                 if (port_is_bonding_slave(pi)) {
2151                         printf("Please remove port %d from bonded device.\n", pi);
2152                         continue;
2153                 }
2154
2155                 port = &ports[pi];
2156                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2157                                                 RTE_PORT_HANDLING) == 0)
2158                         continue;
2159
2160                 rte_eth_dev_stop(pi);
2161
2162                 if (rte_atomic16_cmpset(&(port->port_status),
2163                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2164                         printf("Port %d can not be set into stopped\n", pi);
2165                 need_check_link_status = 1;
2166         }
2167         if (need_check_link_status && !no_link_check)
2168                 check_all_ports_link_status(RTE_PORT_ALL);
2169
2170         printf("Done\n");
2171 }
2172
2173 static void
2174 remove_invalid_ports_in(portid_t *array, portid_t *total)
2175 {
2176         portid_t i;
2177         portid_t new_total = 0;
2178
2179         for (i = 0; i < *total; i++)
2180                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2181                         array[new_total] = array[i];
2182                         new_total++;
2183                 }
2184         *total = new_total;
2185 }
2186
2187 static void
2188 remove_invalid_ports(void)
2189 {
2190         remove_invalid_ports_in(ports_ids, &nb_ports);
2191         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2192         nb_cfg_ports = nb_fwd_ports;
2193 }
2194
2195 void
2196 close_port(portid_t pid)
2197 {
2198         portid_t pi;
2199         struct rte_port *port;
2200
2201         if (port_id_is_invalid(pid, ENABLED_WARN))
2202                 return;
2203
2204         printf("Closing ports...\n");
2205
2206         RTE_ETH_FOREACH_DEV(pi) {
2207                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2208                         continue;
2209
2210                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2211                         printf("Please remove port %d from forwarding configuration.\n", pi);
2212                         continue;
2213                 }
2214
2215                 if (port_is_bonding_slave(pi)) {
2216                         printf("Please remove port %d from bonded device.\n", pi);
2217                         continue;
2218                 }
2219
2220                 port = &ports[pi];
2221                 if (rte_atomic16_cmpset(&(port->port_status),
2222                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2223                         printf("Port %d is already closed\n", pi);
2224                         continue;
2225                 }
2226
2227                 if (rte_atomic16_cmpset(&(port->port_status),
2228                         RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
2229                         printf("Port %d is now not stopped\n", pi);
2230                         continue;
2231                 }
2232
2233                 if (port->flow_list)
2234                         port_flow_flush(pi);
2235                 rte_eth_dev_close(pi);
2236
2237                 remove_invalid_ports();
2238
2239                 if (rte_atomic16_cmpset(&(port->port_status),
2240                         RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
2241                         printf("Port %d cannot be set to closed\n", pi);
2242         }
2243
2244         printf("Done\n");
2245 }
2246
2247 void
2248 reset_port(portid_t pid)
2249 {
2250         int diag;
2251         portid_t pi;
2252         struct rte_port *port;
2253
2254         if (port_id_is_invalid(pid, ENABLED_WARN))
2255                 return;
2256
2257         printf("Resetting ports...\n");
2258
2259         RTE_ETH_FOREACH_DEV(pi) {
2260                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2261                         continue;
2262
2263                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2264                         printf("Please remove port %d from forwarding "
2265                                "configuration.\n", pi);
2266                         continue;
2267                 }
2268
2269                 if (port_is_bonding_slave(pi)) {
2270                         printf("Please remove port %d from bonded device.\n",
2271                                pi);
2272                         continue;
2273                 }
2274
2275                 diag = rte_eth_dev_reset(pi);
2276                 if (diag == 0) {
2277                         port = &ports[pi];
2278                         port->need_reconfig = 1;
2279                         port->need_reconfig_queues = 1;
2280                 } else {
2281                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2282                 }
2283         }
2284
2285         printf("Done\n");
2286 }
2287
2288 void
2289 attach_port(char *identifier)
2290 {
2291         portid_t pi;
2292         struct rte_dev_iterator iterator;
2293
2294         printf("Attaching a new port...\n");
2295
2296         if (identifier == NULL) {
2297                 printf("Invalid parameters are specified\n");
2298                 return;
2299         }
2300
2301         if (rte_dev_probe(identifier) != 0) {
2302                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2303                 return;
2304         }
2305
2306         /* first attach mode: event */
2307         if (setup_on_probe_event) {
2308                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2309                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2310                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2311                                         ports[pi].need_setup != 0)
2312                                 setup_attached_port(pi);
2313                 return;
2314         }
2315
2316         /* second attach mode: iterator */
2317         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2318                 /* setup ports matching the devargs used for probing */
2319                 if (port_is_forwarding(pi))
2320                         continue; /* port was already attached before */
2321                 setup_attached_port(pi);
2322         }
2323 }
2324
2325 static void
2326 setup_attached_port(portid_t pi)
2327 {
2328         unsigned int socket_id;
2329
2330         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2331         /* if socket_id is invalid, set to the first available socket. */
2332         if (check_socket_id(socket_id) < 0)
2333                 socket_id = socket_ids[0];
2334         reconfig(pi, socket_id);
2335         rte_eth_promiscuous_enable(pi);
2336
2337         ports_ids[nb_ports++] = pi;
2338         fwd_ports_ids[nb_fwd_ports++] = pi;
2339         nb_cfg_ports = nb_fwd_ports;
2340         ports[pi].need_setup = 0;
2341         ports[pi].port_status = RTE_PORT_STOPPED;
2342
2343         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2344         printf("Done\n");
2345 }
2346
2347 void
2348 detach_port_device(portid_t port_id)
2349 {
2350         struct rte_device *dev;
2351         portid_t sibling;
2352
2353         printf("Removing a device...\n");
2354
2355         dev = rte_eth_devices[port_id].device;
2356         if (dev == NULL) {
2357                 printf("Device already removed\n");
2358                 return;
2359         }
2360
2361         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2362                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2363                         printf("Port not stopped\n");
2364                         return;
2365                 }
2366                 printf("Port was not closed\n");
2367                 if (ports[port_id].flow_list)
2368                         port_flow_flush(port_id);
2369         }
2370
2371         if (rte_dev_remove(dev) != 0) {
2372                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2373                 return;
2374         }
2375
2376         for (sibling = 0; sibling < RTE_MAX_ETHPORTS; sibling++) {
2377                 if (rte_eth_devices[sibling].device != dev)
2378                         continue;
2379                 /* reset mapping between old ports and removed device */
2380                 rte_eth_devices[sibling].device = NULL;
2381                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2382                         /* sibling ports are forced to be closed */
2383                         ports[sibling].port_status = RTE_PORT_CLOSED;
2384                         printf("Port %u is closed\n", sibling);
2385                 }
2386         }
2387
2388         remove_invalid_ports();
2389
2390         printf("Device of port %u is detached\n", port_id);
2391         printf("Now total ports is %d\n", nb_ports);
2392         printf("Done\n");
2393         return;
2394 }
2395
2396 void
2397 pmd_test_exit(void)
2398 {
2399         struct rte_device *device;
2400         portid_t pt_id;
2401         int ret;
2402
2403         if (test_done == 0)
2404                 stop_packet_forwarding();
2405
2406         if (ports != NULL) {
2407                 no_link_check = 1;
2408                 RTE_ETH_FOREACH_DEV(pt_id) {
2409                         printf("\nShutting down port %d...\n", pt_id);
2410                         fflush(stdout);
2411                         stop_port(pt_id);
2412                         close_port(pt_id);
2413
2414                         /*
2415                          * This is a workaround to fix a virtio-user issue that
2416                          * requires to call clean-up routine to remove existing
2417                          * socket.
2418                          * This workaround valid only for testpmd, needs a fix
2419                          * valid for all applications.
2420                          * TODO: Implement proper resource cleanup
2421                          */
2422                         device = rte_eth_devices[pt_id].device;
2423                         if (device && !strcmp(device->driver->name, "net_virtio_user"))
2424                                 detach_port_device(pt_id);
2425                 }
2426         }
2427
2428         if (hot_plug) {
2429                 ret = rte_dev_event_monitor_stop();
2430                 if (ret) {
2431                         RTE_LOG(ERR, EAL,
2432                                 "fail to stop device event monitor.");
2433                         return;
2434                 }
2435
2436                 ret = rte_dev_event_callback_unregister(NULL,
2437                         eth_dev_event_callback, NULL);
2438                 if (ret < 0) {
2439                         RTE_LOG(ERR, EAL,
2440                                 "fail to unregister device event callback.\n");
2441                         return;
2442                 }
2443
2444                 ret = rte_dev_hotplug_handle_disable();
2445                 if (ret) {
2446                         RTE_LOG(ERR, EAL,
2447                                 "fail to disable hotplug handling.\n");
2448                         return;
2449                 }
2450         }
2451
2452         printf("\nBye...\n");
2453 }
2454
2455 typedef void (*cmd_func_t)(void);
2456 struct pmd_test_command {
2457         const char *cmd_name;
2458         cmd_func_t cmd_func;
2459 };
2460
2461 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0]))
2462
2463 /* Check the link status of all ports in up to 9s, and print them finally */
2464 static void
2465 check_all_ports_link_status(uint32_t port_mask)
2466 {
2467 #define CHECK_INTERVAL 100 /* 100ms */
2468 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
2469         portid_t portid;
2470         uint8_t count, all_ports_up, print_flag = 0;
2471         struct rte_eth_link link;
2472
2473         printf("Checking link statuses...\n");
2474         fflush(stdout);
2475         for (count = 0; count <= MAX_CHECK_TIME; count++) {
2476                 all_ports_up = 1;
2477                 RTE_ETH_FOREACH_DEV(portid) {
2478                         if ((port_mask & (1 << portid)) == 0)
2479                                 continue;
2480                         memset(&link, 0, sizeof(link));
2481                         rte_eth_link_get_nowait(portid, &link);
2482                         /* print link status if flag set */
2483                         if (print_flag == 1) {
2484                                 if (link.link_status)
2485                                         printf(
2486                                         "Port%d Link Up. speed %u Mbps- %s\n",
2487                                         portid, link.link_speed,
2488                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
2489                                         ("full-duplex") : ("half-duplex\n"));
2490                                 else
2491                                         printf("Port %d Link Down\n", portid);
2492                                 continue;
2493                         }
2494                         /* clear all_ports_up flag if any link down */
2495                         if (link.link_status == ETH_LINK_DOWN) {
2496                                 all_ports_up = 0;
2497                                 break;
2498                         }
2499                 }
2500                 /* after finally printing all link status, get out */
2501                 if (print_flag == 1)
2502                         break;
2503
2504                 if (all_ports_up == 0) {
2505                         fflush(stdout);
2506                         rte_delay_ms(CHECK_INTERVAL);
2507                 }
2508
2509                 /* set the print_flag if all ports up or timeout */
2510                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
2511                         print_flag = 1;
2512                 }
2513
2514                 if (lsc_interrupt)
2515                         break;
2516         }
2517 }
2518
2519 static void
2520 rmv_event_callback(void *arg)
2521 {
2522         int need_to_start = 0;
2523         int org_no_link_check = no_link_check;
2524         portid_t port_id = (intptr_t)arg;
2525
2526         RTE_ETH_VALID_PORTID_OR_RET(port_id);
2527
2528         if (!test_done && port_is_forwarding(port_id)) {
2529                 need_to_start = 1;
2530                 stop_packet_forwarding();
2531         }
2532         no_link_check = 1;
2533         stop_port(port_id);
2534         no_link_check = org_no_link_check;
2535         close_port(port_id);
2536         detach_port_device(port_id);
2537         if (need_to_start)
2538                 start_packet_forwarding(0);
2539 }
2540
2541 /* This function is used by the interrupt thread */
2542 static int
2543 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
2544                   void *ret_param)
2545 {
2546         RTE_SET_USED(param);
2547         RTE_SET_USED(ret_param);
2548
2549         if (type >= RTE_ETH_EVENT_MAX) {
2550                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
2551                         port_id, __func__, type);
2552                 fflush(stderr);
2553         } else if (event_print_mask & (UINT32_C(1) << type)) {
2554                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
2555                         eth_event_desc[type]);
2556                 fflush(stdout);
2557         }
2558
2559         switch (type) {
2560         case RTE_ETH_EVENT_NEW:
2561                 ports[port_id].need_setup = 1;
2562                 ports[port_id].port_status = RTE_PORT_HANDLING;
2563                 break;
2564         case RTE_ETH_EVENT_INTR_RMV:
2565                 if (port_id_is_invalid(port_id, DISABLED_WARN))
2566                         break;
2567                 if (rte_eal_alarm_set(100000,
2568                                 rmv_event_callback, (void *)(intptr_t)port_id))
2569                         fprintf(stderr, "Could not set up deferred device removal\n");
2570                 break;
2571         default:
2572                 break;
2573         }
2574         return 0;
2575 }
2576
2577 static int
2578 register_eth_event_callback(void)
2579 {
2580         int ret;
2581         enum rte_eth_event_type event;
2582
2583         for (event = RTE_ETH_EVENT_UNKNOWN;
2584                         event < RTE_ETH_EVENT_MAX; event++) {
2585                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
2586                                 event,
2587                                 eth_event_callback,
2588                                 NULL);
2589                 if (ret != 0) {
2590                         TESTPMD_LOG(ERR, "Failed to register callback for "
2591                                         "%s event\n", eth_event_desc[event]);
2592                         return -1;
2593                 }
2594         }
2595
2596         return 0;
2597 }
2598
2599 /* This function is used by the interrupt thread */
2600 static void
2601 eth_dev_event_callback(const char *device_name, enum rte_dev_event_type type,
2602                              __rte_unused void *arg)
2603 {
2604         uint16_t port_id;
2605         int ret;
2606
2607         if (type >= RTE_DEV_EVENT_MAX) {
2608                 fprintf(stderr, "%s called upon invalid event %d\n",
2609                         __func__, type);
2610                 fflush(stderr);
2611         }
2612
2613         switch (type) {
2614         case RTE_DEV_EVENT_REMOVE:
2615                 RTE_LOG(ERR, EAL, "The device: %s has been removed!\n",
2616                         device_name);
2617                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
2618                 if (ret) {
2619                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
2620                                 device_name);
2621                         return;
2622                 }
2623                 rmv_event_callback((void *)(intptr_t)port_id);
2624                 break;
2625         case RTE_DEV_EVENT_ADD:
2626                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
2627                         device_name);
2628                 /* TODO: After finish kernel driver binding,
2629                  * begin to attach port.
2630                  */
2631                 break;
2632         default:
2633                 break;
2634         }
2635 }
2636
2637 static int
2638 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2639 {
2640         uint16_t i;
2641         int diag;
2642         uint8_t mapping_found = 0;
2643
2644         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
2645                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
2646                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
2647                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
2648                                         tx_queue_stats_mappings[i].queue_id,
2649                                         tx_queue_stats_mappings[i].stats_counter_id);
2650                         if (diag != 0)
2651                                 return diag;
2652                         mapping_found = 1;
2653                 }
2654         }
2655         if (mapping_found)
2656                 port->tx_queue_stats_mapping_enabled = 1;
2657         return 0;
2658 }
2659
2660 static int
2661 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2662 {
2663         uint16_t i;
2664         int diag;
2665         uint8_t mapping_found = 0;
2666
2667         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
2668                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
2669                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
2670                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
2671                                         rx_queue_stats_mappings[i].queue_id,
2672                                         rx_queue_stats_mappings[i].stats_counter_id);
2673                         if (diag != 0)
2674                                 return diag;
2675                         mapping_found = 1;
2676                 }
2677         }
2678         if (mapping_found)
2679                 port->rx_queue_stats_mapping_enabled = 1;
2680         return 0;
2681 }
2682
2683 static void
2684 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
2685 {
2686         int diag = 0;
2687
2688         diag = set_tx_queue_stats_mapping_registers(pi, port);
2689         if (diag != 0) {
2690                 if (diag == -ENOTSUP) {
2691                         port->tx_queue_stats_mapping_enabled = 0;
2692                         printf("TX queue stats mapping not supported port id=%d\n", pi);
2693                 }
2694                 else
2695                         rte_exit(EXIT_FAILURE,
2696                                         "set_tx_queue_stats_mapping_registers "
2697                                         "failed for port id=%d diag=%d\n",
2698                                         pi, diag);
2699         }
2700
2701         diag = set_rx_queue_stats_mapping_registers(pi, port);
2702         if (diag != 0) {
2703                 if (diag == -ENOTSUP) {
2704                         port->rx_queue_stats_mapping_enabled = 0;
2705                         printf("RX queue stats mapping not supported port id=%d\n", pi);
2706                 }
2707                 else
2708                         rte_exit(EXIT_FAILURE,
2709                                         "set_rx_queue_stats_mapping_registers "
2710                                         "failed for port id=%d diag=%d\n",
2711                                         pi, diag);
2712         }
2713 }
2714
2715 static void
2716 rxtx_port_config(struct rte_port *port)
2717 {
2718         uint16_t qid;
2719
2720         for (qid = 0; qid < nb_rxq; qid++) {
2721                 port->rx_conf[qid] = port->dev_info.default_rxconf;
2722
2723                 /* Check if any Rx parameters have been passed */
2724                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
2725                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
2726
2727                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
2728                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
2729
2730                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
2731                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
2732
2733                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
2734                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
2735
2736                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
2737                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
2738
2739                 port->nb_rx_desc[qid] = nb_rxd;
2740         }
2741
2742         for (qid = 0; qid < nb_txq; qid++) {
2743                 port->tx_conf[qid] = port->dev_info.default_txconf;
2744
2745                 /* Check if any Tx parameters have been passed */
2746                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
2747                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
2748
2749                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
2750                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
2751
2752                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
2753                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
2754
2755                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
2756                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
2757
2758                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
2759                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
2760
2761                 port->nb_tx_desc[qid] = nb_txd;
2762         }
2763 }
2764
2765 void
2766 init_port_config(void)
2767 {
2768         portid_t pid;
2769         struct rte_port *port;
2770
2771         RTE_ETH_FOREACH_DEV(pid) {
2772                 port = &ports[pid];
2773                 port->dev_conf.fdir_conf = fdir_conf;
2774                 rte_eth_dev_info_get(pid, &port->dev_info);
2775                 if (nb_rxq > 1) {
2776                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2777                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
2778                                 rss_hf & port->dev_info.flow_type_rss_offloads;
2779                 } else {
2780                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2781                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
2782                 }
2783
2784                 if (port->dcb_flag == 0) {
2785                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
2786                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
2787                         else
2788                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
2789                 }
2790
2791                 rxtx_port_config(port);
2792
2793                 rte_eth_macaddr_get(pid, &port->eth_addr);
2794
2795                 map_port_queue_stats_mapping_registers(pid, port);
2796 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
2797                 rte_pmd_ixgbe_bypass_init(pid);
2798 #endif
2799
2800                 if (lsc_interrupt &&
2801                     (rte_eth_devices[pid].data->dev_flags &
2802                      RTE_ETH_DEV_INTR_LSC))
2803                         port->dev_conf.intr_conf.lsc = 1;
2804                 if (rmv_interrupt &&
2805                     (rte_eth_devices[pid].data->dev_flags &
2806                      RTE_ETH_DEV_INTR_RMV))
2807                         port->dev_conf.intr_conf.rmv = 1;
2808         }
2809 }
2810
2811 void set_port_slave_flag(portid_t slave_pid)
2812 {
2813         struct rte_port *port;
2814
2815         port = &ports[slave_pid];
2816         port->slave_flag = 1;
2817 }
2818
2819 void clear_port_slave_flag(portid_t slave_pid)
2820 {
2821         struct rte_port *port;
2822
2823         port = &ports[slave_pid];
2824         port->slave_flag = 0;
2825 }
2826
2827 uint8_t port_is_bonding_slave(portid_t slave_pid)
2828 {
2829         struct rte_port *port;
2830
2831         port = &ports[slave_pid];
2832         if ((rte_eth_devices[slave_pid].data->dev_flags &
2833             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
2834                 return 1;
2835         return 0;
2836 }
2837
2838 const uint16_t vlan_tags[] = {
2839                 0,  1,  2,  3,  4,  5,  6,  7,
2840                 8,  9, 10, 11,  12, 13, 14, 15,
2841                 16, 17, 18, 19, 20, 21, 22, 23,
2842                 24, 25, 26, 27, 28, 29, 30, 31
2843 };
2844
2845 static  int
2846 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
2847                  enum dcb_mode_enable dcb_mode,
2848                  enum rte_eth_nb_tcs num_tcs,
2849                  uint8_t pfc_en)
2850 {
2851         uint8_t i;
2852         int32_t rc;
2853         struct rte_eth_rss_conf rss_conf;
2854
2855         /*
2856          * Builds up the correct configuration for dcb+vt based on the vlan tags array
2857          * given above, and the number of traffic classes available for use.
2858          */
2859         if (dcb_mode == DCB_VT_ENABLED) {
2860                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
2861                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
2862                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
2863                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
2864
2865                 /* VMDQ+DCB RX and TX configurations */
2866                 vmdq_rx_conf->enable_default_pool = 0;
2867                 vmdq_rx_conf->default_pool = 0;
2868                 vmdq_rx_conf->nb_queue_pools =
2869                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
2870                 vmdq_tx_conf->nb_queue_pools =
2871                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
2872
2873                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
2874                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
2875                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
2876                         vmdq_rx_conf->pool_map[i].pools =
2877                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
2878                 }
2879                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
2880                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
2881                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
2882                 }
2883
2884                 /* set DCB mode of RX and TX of multiple queues */
2885                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB;
2886                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
2887         } else {
2888                 struct rte_eth_dcb_rx_conf *rx_conf =
2889                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
2890                 struct rte_eth_dcb_tx_conf *tx_conf =
2891                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
2892
2893                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
2894                 if (rc != 0)
2895                         return rc;
2896
2897                 rx_conf->nb_tcs = num_tcs;
2898                 tx_conf->nb_tcs = num_tcs;
2899
2900                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
2901                         rx_conf->dcb_tc[i] = i % num_tcs;
2902                         tx_conf->dcb_tc[i] = i % num_tcs;
2903                 }
2904
2905                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS;
2906                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
2907                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
2908         }
2909
2910         if (pfc_en)
2911                 eth_conf->dcb_capability_en =
2912                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
2913         else
2914                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
2915
2916         return 0;
2917 }
2918
2919 int
2920 init_port_dcb_config(portid_t pid,
2921                      enum dcb_mode_enable dcb_mode,
2922                      enum rte_eth_nb_tcs num_tcs,
2923                      uint8_t pfc_en)
2924 {
2925         struct rte_eth_conf port_conf;
2926         struct rte_port *rte_port;
2927         int retval;
2928         uint16_t i;
2929
2930         rte_port = &ports[pid];
2931
2932         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
2933         /* Enter DCB configuration status */
2934         dcb_config = 1;
2935
2936         port_conf.rxmode = rte_port->dev_conf.rxmode;
2937         port_conf.txmode = rte_port->dev_conf.txmode;
2938
2939         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
2940         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
2941         if (retval < 0)
2942                 return retval;
2943         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
2944
2945         /* re-configure the device . */
2946         rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
2947
2948         rte_eth_dev_info_get(pid, &rte_port->dev_info);
2949
2950         /* If dev_info.vmdq_pool_base is greater than 0,
2951          * the queue id of vmdq pools is started after pf queues.
2952          */
2953         if (dcb_mode == DCB_VT_ENABLED &&
2954             rte_port->dev_info.vmdq_pool_base > 0) {
2955                 printf("VMDQ_DCB multi-queue mode is nonsensical"
2956                         " for port %d.", pid);
2957                 return -1;
2958         }
2959
2960         /* Assume the ports in testpmd have the same dcb capability
2961          * and has the same number of rxq and txq in dcb mode
2962          */
2963         if (dcb_mode == DCB_VT_ENABLED) {
2964                 if (rte_port->dev_info.max_vfs > 0) {
2965                         nb_rxq = rte_port->dev_info.nb_rx_queues;
2966                         nb_txq = rte_port->dev_info.nb_tx_queues;
2967                 } else {
2968                         nb_rxq = rte_port->dev_info.max_rx_queues;
2969                         nb_txq = rte_port->dev_info.max_tx_queues;
2970                 }
2971         } else {
2972                 /*if vt is disabled, use all pf queues */
2973                 if (rte_port->dev_info.vmdq_pool_base == 0) {
2974                         nb_rxq = rte_port->dev_info.max_rx_queues;
2975                         nb_txq = rte_port->dev_info.max_tx_queues;
2976                 } else {
2977                         nb_rxq = (queueid_t)num_tcs;
2978                         nb_txq = (queueid_t)num_tcs;
2979
2980                 }
2981         }
2982         rx_free_thresh = 64;
2983
2984         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
2985
2986         rxtx_port_config(rte_port);
2987         /* VLAN filter */
2988         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
2989         for (i = 0; i < RTE_DIM(vlan_tags); i++)
2990                 rx_vft_set(pid, vlan_tags[i], 1);
2991
2992         rte_eth_macaddr_get(pid, &rte_port->eth_addr);
2993         map_port_queue_stats_mapping_registers(pid, rte_port);
2994
2995         rte_port->dcb_flag = 1;
2996
2997         return 0;
2998 }
2999
3000 static void
3001 init_port(void)
3002 {
3003         /* Configuration of Ethernet ports. */
3004         ports = rte_zmalloc("testpmd: ports",
3005                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3006                             RTE_CACHE_LINE_SIZE);
3007         if (ports == NULL) {
3008                 rte_exit(EXIT_FAILURE,
3009                                 "rte_zmalloc(%d struct rte_port) failed\n",
3010                                 RTE_MAX_ETHPORTS);
3011         }
3012
3013         /* Initialize ports NUMA structures */
3014         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3015         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3016         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3017 }
3018
3019 static void
3020 force_quit(void)
3021 {
3022         pmd_test_exit();
3023         prompt_exit();
3024 }
3025
3026 static void
3027 print_stats(void)
3028 {
3029         uint8_t i;
3030         const char clr[] = { 27, '[', '2', 'J', '\0' };
3031         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3032
3033         /* Clear screen and move to top left */
3034         printf("%s%s", clr, top_left);
3035
3036         printf("\nPort statistics ====================================");
3037         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3038                 nic_stats_display(fwd_ports_ids[i]);
3039 }
3040
3041 static void
3042 signal_handler(int signum)
3043 {
3044         if (signum == SIGINT || signum == SIGTERM) {
3045                 printf("\nSignal %d received, preparing to exit...\n",
3046                                 signum);
3047 #ifdef RTE_LIBRTE_PDUMP
3048                 /* uninitialize packet capture framework */
3049                 rte_pdump_uninit();
3050 #endif
3051 #ifdef RTE_LIBRTE_LATENCY_STATS
3052                 rte_latencystats_uninit();
3053 #endif
3054                 force_quit();
3055                 /* Set flag to indicate the force termination. */
3056                 f_quit = 1;
3057                 /* exit with the expected status */
3058                 signal(signum, SIG_DFL);
3059                 kill(getpid(), signum);
3060         }
3061 }
3062
3063 int
3064 main(int argc, char** argv)
3065 {
3066         int diag;
3067         portid_t port_id;
3068         uint16_t count;
3069         int ret;
3070
3071         signal(SIGINT, signal_handler);
3072         signal(SIGTERM, signal_handler);
3073
3074         diag = rte_eal_init(argc, argv);
3075         if (diag < 0)
3076                 rte_panic("Cannot init EAL\n");
3077
3078         testpmd_logtype = rte_log_register("testpmd");
3079         if (testpmd_logtype < 0)
3080                 rte_panic("Cannot register log type");
3081         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3082
3083         ret = register_eth_event_callback();
3084         if (ret != 0)
3085                 rte_panic("Cannot register for ethdev events");
3086
3087 #ifdef RTE_LIBRTE_PDUMP
3088         /* initialize packet capture framework */
3089         rte_pdump_init(NULL);
3090 #endif
3091
3092         count = 0;
3093         RTE_ETH_FOREACH_DEV(port_id) {
3094                 ports_ids[count] = port_id;
3095                 count++;
3096         }
3097         nb_ports = (portid_t) count;
3098         if (nb_ports == 0)
3099                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3100
3101         /* allocate port structures, and init them */
3102         init_port();
3103
3104         set_def_fwd_config();
3105         if (nb_lcores == 0)
3106                 rte_panic("Empty set of forwarding logical cores - check the "
3107                           "core mask supplied in the command parameters\n");
3108
3109         /* Bitrate/latency stats disabled by default */
3110 #ifdef RTE_LIBRTE_BITRATE
3111         bitrate_enabled = 0;
3112 #endif
3113 #ifdef RTE_LIBRTE_LATENCY_STATS
3114         latencystats_enabled = 0;
3115 #endif
3116
3117         /* on FreeBSD, mlockall() is disabled by default */
3118 #ifdef RTE_EXEC_ENV_BSDAPP
3119         do_mlockall = 0;
3120 #else
3121         do_mlockall = 1;
3122 #endif
3123
3124         argc -= diag;
3125         argv += diag;
3126         if (argc > 1)
3127                 launch_args_parse(argc, argv);
3128
3129         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3130                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3131                         strerror(errno));
3132         }
3133
3134         if (tx_first && interactive)
3135                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3136                                 "interactive mode.\n");
3137
3138         if (tx_first && lsc_interrupt) {
3139                 printf("Warning: lsc_interrupt needs to be off when "
3140                                 " using tx_first. Disabling.\n");
3141                 lsc_interrupt = 0;
3142         }
3143
3144         if (!nb_rxq && !nb_txq)
3145                 printf("Warning: Either rx or tx queues should be non-zero\n");
3146
3147         if (nb_rxq > 1 && nb_rxq > nb_txq)
3148                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3149                        "but nb_txq=%d will prevent to fully test it.\n",
3150                        nb_rxq, nb_txq);
3151
3152         init_config();
3153
3154         if (hot_plug) {
3155                 ret = rte_dev_hotplug_handle_enable();
3156                 if (ret) {
3157                         RTE_LOG(ERR, EAL,
3158                                 "fail to enable hotplug handling.");
3159                         return -1;
3160                 }
3161
3162                 ret = rte_dev_event_monitor_start();
3163                 if (ret) {
3164                         RTE_LOG(ERR, EAL,
3165                                 "fail to start device event monitoring.");
3166                         return -1;
3167                 }
3168
3169                 ret = rte_dev_event_callback_register(NULL,
3170                         eth_dev_event_callback, NULL);
3171                 if (ret) {
3172                         RTE_LOG(ERR, EAL,
3173                                 "fail  to register device event callback\n");
3174                         return -1;
3175                 }
3176         }
3177
3178         if (start_port(RTE_PORT_ALL) != 0)
3179                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3180
3181         /* set all ports to promiscuous mode by default */
3182         RTE_ETH_FOREACH_DEV(port_id)
3183                 rte_eth_promiscuous_enable(port_id);
3184
3185         /* Init metrics library */
3186         rte_metrics_init(rte_socket_id());
3187
3188 #ifdef RTE_LIBRTE_LATENCY_STATS
3189         if (latencystats_enabled != 0) {
3190                 int ret = rte_latencystats_init(1, NULL);
3191                 if (ret)
3192                         printf("Warning: latencystats init()"
3193                                 " returned error %d\n", ret);
3194                 printf("Latencystats running on lcore %d\n",
3195                         latencystats_lcore_id);
3196         }
3197 #endif
3198
3199         /* Setup bitrate stats */
3200 #ifdef RTE_LIBRTE_BITRATE
3201         if (bitrate_enabled != 0) {
3202                 bitrate_data = rte_stats_bitrate_create();
3203                 if (bitrate_data == NULL)
3204                         rte_exit(EXIT_FAILURE,
3205                                 "Could not allocate bitrate data.\n");
3206                 rte_stats_bitrate_reg(bitrate_data);
3207         }
3208 #endif
3209
3210 #ifdef RTE_LIBRTE_CMDLINE
3211         if (strlen(cmdline_filename) != 0)
3212                 cmdline_read_from_file(cmdline_filename);
3213
3214         if (interactive == 1) {
3215                 if (auto_start) {
3216                         printf("Start automatic packet forwarding\n");
3217                         start_packet_forwarding(0);
3218                 }
3219                 prompt();
3220                 pmd_test_exit();
3221         } else
3222 #endif
3223         {
3224                 char c;
3225                 int rc;
3226
3227                 f_quit = 0;
3228
3229                 printf("No commandline core given, start packet forwarding\n");
3230                 start_packet_forwarding(tx_first);
3231                 if (stats_period != 0) {
3232                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3233                         uint64_t timer_period;
3234
3235                         /* Convert to number of cycles */
3236                         timer_period = stats_period * rte_get_timer_hz();
3237
3238                         while (f_quit == 0) {
3239                                 cur_time = rte_get_timer_cycles();
3240                                 diff_time += cur_time - prev_time;
3241
3242                                 if (diff_time >= timer_period) {
3243                                         print_stats();
3244                                         /* Reset the timer */
3245                                         diff_time = 0;
3246                                 }
3247                                 /* Sleep to avoid unnecessary checks */
3248                                 prev_time = cur_time;
3249                                 sleep(1);
3250                         }
3251                 }
3252
3253                 printf("Press enter to exit\n");
3254                 rc = read(0, &c, 1);
3255                 pmd_test_exit();
3256                 if (rc < 0)
3257                         return 1;
3258         }
3259
3260         return 0;
3261 }