app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_malloc_heap.h>
  31 #include <rte_memory.h>
  32 #include <rte_memcpy.h>
  33 #include <rte_launch.h>
  34 #include <rte_eal.h>
  35 #include <rte_alarm.h>
  36 #include <rte_per_lcore.h>
  37 #include <rte_lcore.h>
  38 #include <rte_atomic.h>
  39 #include <rte_branch_prediction.h>
  40 #include <rte_mempool.h>
  41 #include <rte_malloc.h>
  42 #include <rte_mbuf.h>
  43 #include <rte_mbuf_pool_ops.h>
  44 #include <rte_interrupts.h>
  45 #include <rte_pci.h>
  46 #include <rte_ether.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_dev.h>
  49 #include <rte_string_fns.h>
  50 #ifdef RTE_LIBRTE_IXGBE_PMD
  51 #include <rte_pmd_ixgbe.h>
  52 #endif
  53 #ifdef RTE_LIBRTE_PDUMP
  54 #include <rte_pdump.h>
  55 #endif
  56 #include <rte_flow.h>
  57 #include <rte_metrics.h>
  58 #ifdef RTE_LIBRTE_BITRATE
  59 #include <rte_bitrate.h>
  60 #endif
  61 #ifdef RTE_LIBRTE_LATENCY_STATS
  62 #include <rte_latencystats.h>
  63 #endif
  64
  65 #include "testpmd.h"
  66
  67 #ifndef MAP_HUGETLB
  68 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  69 #define HUGE_FLAG (0x40000)
  70 #else
  71 #define HUGE_FLAG MAP_HUGETLB
  72 #endif
  73
  74 #ifndef MAP_HUGE_SHIFT
  75 /* older kernels (or FreeBSD) will not have this define */
  76 #define HUGE_SHIFT (26)
  77 #else
  78 #define HUGE_SHIFT MAP_HUGE_SHIFT
  79 #endif
  80
  81 #define EXTMEM_HEAP_NAME "extmem"
  82
  83 uint16_t verbose_level = 0; /**< Silent by default. */
  84 int testpmd_logtype; /**< Log type for testpmd logs */
  85
  86 /* use master core for command line ? */
  87 uint8_t interactive = 0;
  88 uint8_t auto_start = 0;
  89 uint8_t tx_first;
  90 char cmdline_filename[PATH_MAX] = {0};
  91
  92 /*
  93  * NUMA support configuration.
  94  * When set, the NUMA support attempts to dispatch the allocation of the
  95  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  96  * probed ports among the CPU sockets 0 and 1.
  97  * Otherwise, all memory is allocated from CPU socket 0.
  98  */
  99 uint8_t numa_support = 1; /**< numa enabled by default */
 100
 101 /*
 102  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 103  * not configured.
 104  */
 105 uint8_t socket_num = UMA_NO_CONFIG;
 106
 107 /*
 108  * Select mempool allocation type:
 109  * - native: use regular DPDK memory
 110  * - anon: use regular DPDK memory to create mempool, but populate using
 111  *         anonymous memory (may not be IOVA-contiguous)
 112  * - xmem: use externally allocated hugepage memory
 113  */
 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 115
 116 /*
 117  * Store specified sockets on which memory pool to be used by ports
 118  * is allocated.
 119  */
 120 uint8_t port_numa[RTE_MAX_ETHPORTS];
 121
 122 /*
 123  * Store specified sockets on which RX ring to be used by ports
 124  * is allocated.
 125  */
 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 127
 128 /*
 129  * Store specified sockets on which TX ring to be used by ports
 130  * is allocated.
 131  */
 132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 133
 134 /*
 135  * Record the Ethernet address of peer target ports to which packets are
 136  * forwarded.
 137  * Must be instantiated with the ethernet addresses of peer traffic generator
 138  * ports.
 139  */
 140 struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 141 portid_t nb_peer_eth_addrs = 0;
 142
 143 /*
 144  * Probed Target Environment.
 145  */
 146 struct rte_port *ports;        /**< For all probed ethernet ports. */
 147 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 149 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 150
 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 152
 153 /*
 154  * Test Forwarding Configuration.
 155  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 156  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 157  */
 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 160 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 161 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 162
 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 165
 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 167 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 168
 169 /*
 170  * Forwarding engines.
 171  */
 172 struct fwd_engine * fwd_engines[] = {
 173         &io_fwd_engine,
 174         &mac_fwd_engine,
 175         &mac_swap_engine,
 176         &flow_gen_engine,
 177         &rx_only_engine,
 178         &tx_only_engine,
 179         &csum_fwd_engine,
 180         &icmp_echo_engine,
 181         &noisy_vnf_engine,
 182 #if defined RTE_LIBRTE_PMD_SOFTNIC
 183         &softnic_fwd_engine,
 184 #endif
 185 #ifdef RTE_LIBRTE_IEEE1588
 186         &ieee1588_fwd_engine,
 187 #endif
 188         NULL,
 189 };
 190
 191 struct fwd_config cur_fwd_config;
 192 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 193 uint32_t retry_enabled;
 194 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 195 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 196
 197 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
 198 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 199                                       * specified on command-line. */
 200 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 201
 202 /*
 203  * In container, it cannot terminate the process which running with 'stats-period'
 204  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 205  */
 206 uint8_t f_quit;
 207
 208 /*
 209  * Configuration of packet segments used by the "txonly" processing engine.
 210  */
 211 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 212 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 213         TXONLY_DEF_PACKET_LEN,
 214 };
 215 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 216
 217 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 218 /**< Split policy for packets to TX. */
 219
 220 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 221 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 222
 223 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 224 uint8_t dcb_config = 0;
 225
 226 /* Whether the dcb is in testing status */
 227 uint8_t dcb_test = 0;
 228
 229 /*
 230  * Configurable number of RX/TX queues.
 231  */
 232 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 233 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 234
 235 /*
 236  * Configurable number of RX/TX ring descriptors.
 237  * Defaults are supplied by drivers via ethdev.
 238  */
 239 #define RTE_TEST_RX_DESC_DEFAULT 0
 240 #define RTE_TEST_TX_DESC_DEFAULT 0
 241 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 242 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 243
 244 #define RTE_PMD_PARAM_UNSET -1
 245 /*
 246  * Configurable values of RX and TX ring threshold registers.
 247  */
 248
 249 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 250 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 251 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 252
 253 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 254 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 255 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 256
 257 /*
 258  * Configurable value of RX free threshold.
 259  */
 260 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 261
 262 /*
 263  * Configurable value of RX drop enable.
 264  */
 265 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 266
 267 /*
 268  * Configurable value of TX free threshold.
 269  */
 270 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 271
 272 /*
 273  * Configurable value of TX RS bit threshold.
 274  */
 275 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 276
 277 /*
 278  * Configurable value of buffered packets before sending.
 279  */
 280 uint16_t noisy_tx_sw_bufsz;
 281
 282 /*
 283  * Configurable value of packet buffer timeout.
 284  */
 285 uint16_t noisy_tx_sw_buf_flush_time;
 286
 287 /*
 288  * Configurable value for size of VNF internal memory area
 289  * used for simulating noisy neighbour behaviour
 290  */
 291 uint64_t noisy_lkup_mem_sz;
 292
 293 /*
 294  * Configurable value of number of random writes done in
 295  * VNF simulation memory area.
 296  */
 297 uint64_t noisy_lkup_num_writes;
 298
 299 /*
 300  * Configurable value of number of random reads done in
 301  * VNF simulation memory area.
 302  */
 303 uint64_t noisy_lkup_num_reads;
 304
 305 /*
 306  * Configurable value of number of random reads/writes done in
 307  * VNF simulation memory area.
 308  */
 309 uint64_t noisy_lkup_num_reads_writes;
 310
 311 /*
 312  * Receive Side Scaling (RSS) configuration.
 313  */
 314 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 315
 316 /*
 317  * Port topology configuration
 318  */
 319 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 320
 321 /*
 322  * Avoids to flush all the RX streams before starts forwarding.
 323  */
 324 uint8_t no_flush_rx = 0; /* flush by default */
 325
 326 /*
 327  * Flow API isolated mode.
 328  */
 329 uint8_t flow_isolate_all;
 330
 331 /*
 332  * Avoids to check link status when starting/stopping a port.
 333  */
 334 uint8_t no_link_check = 0; /* check by default */
 335
 336 /*
 337  * Enable link status change notification
 338  */
 339 uint8_t lsc_interrupt = 1; /* enabled by default */
 340
 341 /*
 342  * Enable device removal notification.
 343  */
 344 uint8_t rmv_interrupt = 1; /* enabled by default */
 345
 346 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 347
 348 /* After attach, port setup is called on event or by iterator */
 349 bool setup_on_probe_event = true;
 350
 351 /* Pretty printing of ethdev events */
 352 static const char * const eth_event_desc[] = {
 353         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 354         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 355         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 356         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 357         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 358         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 359         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 360         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 361         [RTE_ETH_EVENT_NEW] = "device probed",
 362         [RTE_ETH_EVENT_DESTROY] = "device released",
 363         [RTE_ETH_EVENT_MAX] = NULL,
 364 };
 365
 366 /*
 367  * Display or mask ether events
 368  * Default to all events except VF_MBOX
 369  */
 370 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 371                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 372                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 373                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 374                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 375                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 376                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV);
 377 /*
 378  * Decide if all memory are locked for performance.
 379  */
 380 int do_mlockall = 0;
 381
 382 /*
 383  * NIC bypass mode configuration options.
 384  */
 385
 386 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
 387 /* The NIC bypass watchdog timeout. */
 388 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 389 #endif
 390
 391
 392 #ifdef RTE_LIBRTE_LATENCY_STATS
 393
 394 /*
 395  * Set when latency stats is enabled in the commandline
 396  */
 397 uint8_t latencystats_enabled;
 398
 399 /*
 400  * Lcore ID to serive latency statistics.
 401  */
 402 lcoreid_t latencystats_lcore_id = -1;
 403
 404 #endif
 405
 406 /*
 407  * Ethernet device configuration.
 408  */
 409 struct rte_eth_rxmode rx_mode = {
 410         .max_rx_pkt_len = ETHER_MAX_LEN, /**< Default maximum frame length. */
 411 };
 412
 413 struct rte_eth_txmode tx_mode = {
 414         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 415 };
 416
 417 struct rte_fdir_conf fdir_conf = {
 418         .mode = RTE_FDIR_MODE_NONE,
 419         .pballoc = RTE_FDIR_PBALLOC_64K,
 420         .status = RTE_FDIR_REPORT_STATUS,
 421         .mask = {
 422                 .vlan_tci_mask = 0xFFEF,
 423                 .ipv4_mask     = {
 424                         .src_ip = 0xFFFFFFFF,
 425                         .dst_ip = 0xFFFFFFFF,
 426                 },
 427                 .ipv6_mask     = {
 428                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 429                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 430                 },
 431                 .src_port_mask = 0xFFFF,
 432                 .dst_port_mask = 0xFFFF,
 433                 .mac_addr_byte_mask = 0xFF,
 434                 .tunnel_type_mask = 1,
 435                 .tunnel_id_mask = 0xFFFFFFFF,
 436         },
 437         .drop_queue = 127,
 438 };
 439
 440 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 441
 442 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 443 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 444
 445 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 446 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 447
 448 uint16_t nb_tx_queue_stats_mappings = 0;
 449 uint16_t nb_rx_queue_stats_mappings = 0;
 450
 451 /*
 452  * Display zero values by default for xstats
 453  */
 454 uint8_t xstats_hide_zero;
 455
 456 unsigned int num_sockets = 0;
 457 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 458
 459 #ifdef RTE_LIBRTE_BITRATE
 460 /* Bitrate statistics */
 461 struct rte_stats_bitrates *bitrate_data;
 462 lcoreid_t bitrate_lcore_id;
 463 uint8_t bitrate_enabled;
 464 #endif
 465
 466 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 467 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 468
 469 struct vxlan_encap_conf vxlan_encap_conf = {
 470         .select_ipv4 = 1,
 471         .select_vlan = 0,
 472         .vni = "\x00\x00\x00",
 473         .udp_src = 0,
 474         .udp_dst = RTE_BE16(4789),
 475         .ipv4_src = IPv4(127, 0, 0, 1),
 476         .ipv4_dst = IPv4(255, 255, 255, 255),
 477         .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00"
 478                 "\x00\x00\x00\x00\x00\x00\x00\x01",
 479         .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00"
 480                 "\x00\x00\x00\x00\x00\x00\x11\x11",
 481         .vlan_tci = 0,
 482         .eth_src = "\x00\x00\x00\x00\x00\x00",
 483         .eth_dst = "\xff\xff\xff\xff\xff\xff",
 484 };
 485
 486 struct nvgre_encap_conf nvgre_encap_conf = {
 487         .select_ipv4 = 1,
 488         .select_vlan = 0,
 489         .tni = "\x00\x00\x00",
 490         .ipv4_src = IPv4(127, 0, 0, 1),
 491         .ipv4_dst = IPv4(255, 255, 255, 255),
 492         .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00"
 493                 "\x00\x00\x00\x00\x00\x00\x00\x01",
 494         .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00"
 495                 "\x00\x00\x00\x00\x00\x00\x11\x11",
 496         .vlan_tci = 0,
 497         .eth_src = "\x00\x00\x00\x00\x00\x00",
 498         .eth_dst = "\xff\xff\xff\xff\xff\xff",
 499 };
 500
 501 /* Forward function declarations */
 502 static void setup_attached_port(portid_t pi);
 503 static void map_port_queue_stats_mapping_registers(portid_t pi,
 504                                                    struct rte_port *port);
 505 static void check_all_ports_link_status(uint32_t port_mask);
 506 static int eth_event_callback(portid_t port_id,
 507                               enum rte_eth_event_type type,
 508                               void *param, void *ret_param);
 509 static void dev_event_callback(const char *device_name,
 510                                 enum rte_dev_event_type type,
 511                                 void *param);
 512
 513 /*
 514  * Check if all the ports are started.
 515  * If yes, return positive value. If not, return zero.
 516  */
 517 static int all_ports_started(void);
 518
 519 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 520 uint16_t gso_max_segment_size = ETHER_MAX_LEN - ETHER_CRC_LEN;
 521
 522 /*
 523  * Helper function to check if socket is already discovered.
 524  * If yes, return positive value. If not, return zero.
 525  */
 526 int
 527 new_socket_id(unsigned int socket_id)
 528 {
 529         unsigned int i;
 530
 531         for (i = 0; i < num_sockets; i++) {
 532                 if (socket_ids[i] == socket_id)
 533                         return 0;
 534         }
 535         return 1;
 536 }
 537
 538 /*
 539  * Setup default configuration.
 540  */
 541 static void
 542 set_default_fwd_lcores_config(void)
 543 {
 544         unsigned int i;
 545         unsigned int nb_lc;
 546         unsigned int sock_num;
 547
 548         nb_lc = 0;
 549         for (i = 0; i < RTE_MAX_LCORE; i++) {
 550                 if (!rte_lcore_is_enabled(i))
 551                         continue;
 552                 sock_num = rte_lcore_to_socket_id(i);
 553                 if (new_socket_id(sock_num)) {
 554                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 555                                 rte_exit(EXIT_FAILURE,
 556                                          "Total sockets greater than %u\n",
 557                                          RTE_MAX_NUMA_NODES);
 558                         }
 559                         socket_ids[num_sockets++] = sock_num;
 560                 }
 561                 if (i == rte_get_master_lcore())
 562                         continue;
 563                 fwd_lcores_cpuids[nb_lc++] = i;
 564         }
 565         nb_lcores = (lcoreid_t) nb_lc;
 566         nb_cfg_lcores = nb_lcores;
 567         nb_fwd_lcores = 1;
 568 }
 569
 570 static void
 571 set_def_peer_eth_addrs(void)
 572 {
 573         portid_t i;
 574
 575         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 576                 peer_eth_addrs[i].addr_bytes[0] = ETHER_LOCAL_ADMIN_ADDR;
 577                 peer_eth_addrs[i].addr_bytes[5] = i;
 578         }
 579 }
 580
 581 static void
 582 set_default_fwd_ports_config(void)
 583 {
 584         portid_t pt_id;
 585         int i = 0;
 586
 587         RTE_ETH_FOREACH_DEV(pt_id) {
 588                 fwd_ports_ids[i++] = pt_id;
 589
 590                 /* Update sockets info according to the attached device */
 591                 int socket_id = rte_eth_dev_socket_id(pt_id);
 592                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 593                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 594                                 rte_exit(EXIT_FAILURE,
 595                                          "Total sockets greater than %u\n",
 596                                          RTE_MAX_NUMA_NODES);
 597                         }
 598                         socket_ids[num_sockets++] = socket_id;
 599                 }
 600         }
 601
 602         nb_cfg_ports = nb_ports;
 603         nb_fwd_ports = nb_ports;
 604 }
 605
 606 void
 607 set_def_fwd_config(void)
 608 {
 609         set_default_fwd_lcores_config();
 610         set_def_peer_eth_addrs();
 611         set_default_fwd_ports_config();
 612 }
 613
 614 /* extremely pessimistic estimation of memory required to create a mempool */
 615 static int
 616 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 617 {
 618         unsigned int n_pages, mbuf_per_pg, leftover;
 619         uint64_t total_mem, mbuf_mem, obj_sz;
 620
 621         /* there is no good way to predict how much space the mempool will
 622          * occupy because it will allocate chunks on the fly, and some of those
 623          * will come from default DPDK memory while some will come from our
 624          * external memory, so just assume 128MB will be enough for everyone.
 625          */
 626         uint64_t hdr_mem = 128 << 20;
 627
 628         /* account for possible non-contiguousness */
 629         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 630         if (obj_sz > pgsz) {
 631                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 632                 return -1;
 633         }
 634
 635         mbuf_per_pg = pgsz / obj_sz;
 636         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 637         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 638
 639         mbuf_mem = n_pages * pgsz;
 640
 641         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 642
 643         if (total_mem > SIZE_MAX) {
 644                 TESTPMD_LOG(ERR, "Memory size too big\n");
 645                 return -1;
 646         }
 647         *out = (size_t)total_mem;
 648
 649         return 0;
 650 }
 651
 652 static inline uint32_t
 653 bsf64(uint64_t v)
 654 {
 655         return (uint32_t)__builtin_ctzll(v);
 656 }
 657
 658 static inline uint32_t
 659 log2_u64(uint64_t v)
 660 {
 661         if (v == 0)
 662                 return 0;
 663         v = rte_align64pow2(v);
 664         return bsf64(v);
 665 }
 666
 667 static int
 668 pagesz_flags(uint64_t page_sz)
 669 {
 670         /* as per mmap() manpage, all page sizes are log2 of page size
 671          * shifted by MAP_HUGE_SHIFT
 672          */
 673         int log2 = log2_u64(page_sz);
 674
 675         return (log2 << HUGE_SHIFT);
 676 }
 677
 678 static void *
 679 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 680 {
 681         void *addr;
 682         int flags;
 683
 684         /* allocate anonymous hugepages */
 685         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 686         if (huge)
 687                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 688
 689         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 690         if (addr == MAP_FAILED)
 691                 return NULL;
 692
 693         return addr;
 694 }
 695
 696 struct extmem_param {
 697         void *addr;
 698         size_t len;
 699         size_t pgsz;
 700         rte_iova_t *iova_table;
 701         unsigned int iova_table_len;
 702 };
 703
 704 static int
 705 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 706                 bool huge)
 707 {
 708         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 709                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 710         unsigned int cur_page, n_pages, pgsz_idx;
 711         size_t mem_sz, cur_pgsz;
 712         rte_iova_t *iovas = NULL;
 713         void *addr;
 714         int ret;
 715
 716         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 717                 /* skip anything that is too big */
 718                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 719                         continue;
 720
 721                 cur_pgsz = pgsizes[pgsz_idx];
 722
 723                 /* if we were told not to allocate hugepages, override */
 724                 if (!huge)
 725                         cur_pgsz = sysconf(_SC_PAGESIZE);
 726
 727                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 728                 if (ret < 0) {
 729                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 730                         return -1;
 731                 }
 732
 733                 /* allocate our memory */
 734                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 735
 736                 /* if we couldn't allocate memory with a specified page size,
 737                  * that doesn't mean we can't do it with other page sizes, so
 738                  * try another one.
 739                  */
 740                 if (addr == NULL)
 741                         continue;
 742
 743                 /* store IOVA addresses for every page in this memory area */
 744                 n_pages = mem_sz / cur_pgsz;
 745
 746                 iovas = malloc(sizeof(*iovas) * n_pages);
 747
 748                 if (iovas == NULL) {
 749                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 750                         goto fail;
 751                 }
 752                 /* lock memory if it's not huge pages */
 753                 if (!huge)
 754                         mlock(addr, mem_sz);
 755
 756                 /* populate IOVA addresses */
 757                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 758                         rte_iova_t iova;
 759                         size_t offset;
 760                         void *cur;
 761
 762                         offset = cur_pgsz * cur_page;
 763                         cur = RTE_PTR_ADD(addr, offset);
 764
 765                         /* touch the page before getting its IOVA */
 766                         *(volatile char *)cur = 0;
 767
 768                         iova = rte_mem_virt2iova(cur);
 769
 770                         iovas[cur_page] = iova;
 771                 }
 772
 773                 break;
 774         }
 775         /* if we couldn't allocate anything */
 776         if (iovas == NULL)
 777                 return -1;
 778
 779         param->addr = addr;
 780         param->len = mem_sz;
 781         param->pgsz = cur_pgsz;
 782         param->iova_table = iovas;
 783         param->iova_table_len = n_pages;
 784
 785         return 0;
 786 fail:
 787         if (iovas)
 788                 free(iovas);
 789         if (addr)
 790                 munmap(addr, mem_sz);
 791
 792         return -1;
 793 }
 794
 795 static int
 796 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 797 {
 798         struct extmem_param param;
 799         int socket_id, ret;
 800
 801         memset(&param, 0, sizeof(param));
 802
 803         /* check if our heap exists */
 804         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 805         if (socket_id < 0) {
 806                 /* create our heap */
 807                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 808                 if (ret < 0) {
 809                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 810                         return -1;
 811                 }
 812         }
 813
 814         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 815         if (ret < 0) {
 816                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 817                 return -1;
 818         }
 819
 820         /* we now have a valid memory area, so add it to heap */
 821         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 822                         param.addr, param.len, param.iova_table,
 823                         param.iova_table_len, param.pgsz);
 824
 825         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 826
 827         /* not needed any more */
 828         free(param.iova_table);
 829
 830         if (ret < 0) {
 831                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 832                 munmap(param.addr, param.len);
 833                 return -1;
 834         }
 835
 836         /* success */
 837
 838         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 839                         param.len >> 20);
 840
 841         return 0;
 842 }
 843
 844 /*
 845  * Configuration initialisation done once at init time.
 846  */
 847 static void
 848 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 849                  unsigned int socket_id)
 850 {
 851         char pool_name[RTE_MEMPOOL_NAMESIZE];
 852         struct rte_mempool *rte_mp = NULL;
 853         uint32_t mb_size;
 854
 855         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 856         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
 857
 858         TESTPMD_LOG(INFO,
 859                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 860                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 861
 862         switch (mp_alloc_type) {
 863         case MP_ALLOC_NATIVE:
 864                 {
 865                         /* wrapper to rte_mempool_create() */
 866                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 867                                         rte_mbuf_best_mempool_ops());
 868                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 869                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 870                         break;
 871                 }
 872         case MP_ALLOC_ANON:
 873                 {
 874                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 875                                 mb_size, (unsigned int) mb_mempool_cache,
 876                                 sizeof(struct rte_pktmbuf_pool_private),
 877                                 socket_id, 0);
 878                         if (rte_mp == NULL)
 879                                 goto err;
 880
 881                         if (rte_mempool_populate_anon(rte_mp) == 0) {
 882                                 rte_mempool_free(rte_mp);
 883                                 rte_mp = NULL;
 884                                 goto err;
 885                         }
 886                         rte_pktmbuf_pool_init(rte_mp, NULL);
 887                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
 888                         break;
 889                 }
 890         case MP_ALLOC_XMEM:
 891         case MP_ALLOC_XMEM_HUGE:
 892                 {
 893                         int heap_socket;
 894                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
 895
 896                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
 897                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
 898
 899                         heap_socket =
 900                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 901                         if (heap_socket < 0)
 902                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
 903
 904                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 905                                         rte_mbuf_best_mempool_ops());
 906                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 907                                         mb_mempool_cache, 0, mbuf_seg_size,
 908                                         heap_socket);
 909                         break;
 910                 }
 911         default:
 912                 {
 913                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
 914                 }
 915         }
 916
 917 err:
 918         if (rte_mp == NULL) {
 919                 rte_exit(EXIT_FAILURE,
 920                         "Creation of mbuf pool for socket %u failed: %s\n",
 921                         socket_id, rte_strerror(rte_errno));
 922         } else if (verbose_level > 0) {
 923                 rte_mempool_dump(stdout, rte_mp);
 924         }
 925 }
 926
 927 /*
 928  * Check given socket id is valid or not with NUMA mode,
 929  * if valid, return 0, else return -1
 930  */
 931 static int
 932 check_socket_id(const unsigned int socket_id)
 933 {
 934         static int warning_once = 0;
 935
 936         if (new_socket_id(socket_id)) {
 937                 if (!warning_once && numa_support)
 938                         printf("Warning: NUMA should be configured manually by"
 939                                " using --port-numa-config and"
 940                                " --ring-numa-config parameters along with"
 941                                " --numa.\n");
 942                 warning_once = 1;
 943                 return -1;
 944         }
 945         return 0;
 946 }
 947
 948 /*
 949  * Get the allowed maximum number of RX queues.
 950  * *pid return the port id which has minimal value of
 951  * max_rx_queues in all ports.
 952  */
 953 queueid_t
 954 get_allowed_max_nb_rxq(portid_t *pid)
 955 {
 956         queueid_t allowed_max_rxq = MAX_QUEUE_ID;
 957         portid_t pi;
 958         struct rte_eth_dev_info dev_info;
 959
 960         RTE_ETH_FOREACH_DEV(pi) {
 961                 rte_eth_dev_info_get(pi, &dev_info);
 962                 if (dev_info.max_rx_queues < allowed_max_rxq) {
 963                         allowed_max_rxq = dev_info.max_rx_queues;
 964                         *pid = pi;
 965                 }
 966         }
 967         return allowed_max_rxq;
 968 }
 969
 970 /*
 971  * Check input rxq is valid or not.
 972  * If input rxq is not greater than any of maximum number
 973  * of RX queues of all ports, it is valid.
 974  * if valid, return 0, else return -1
 975  */
 976 int
 977 check_nb_rxq(queueid_t rxq)
 978 {
 979         queueid_t allowed_max_rxq;
 980         portid_t pid = 0;
 981
 982         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
 983         if (rxq > allowed_max_rxq) {
 984                 printf("Fail: input rxq (%u) can't be greater "
 985                        "than max_rx_queues (%u) of port %u\n",
 986                        rxq,
 987                        allowed_max_rxq,
 988                        pid);
 989                 return -1;
 990         }
 991         return 0;
 992 }
 993
 994 /*
 995  * Get the allowed maximum number of TX queues.
 996  * *pid return the port id which has minimal value of
 997  * max_tx_queues in all ports.
 998  */
 999 queueid_t
1000 get_allowed_max_nb_txq(portid_t *pid)
1001 {
1002         queueid_t allowed_max_txq = MAX_QUEUE_ID;
1003         portid_t pi;
1004         struct rte_eth_dev_info dev_info;
1005
1006         RTE_ETH_FOREACH_DEV(pi) {
1007                 rte_eth_dev_info_get(pi, &dev_info);
1008                 if (dev_info.max_tx_queues < allowed_max_txq) {
1009                         allowed_max_txq = dev_info.max_tx_queues;
1010                         *pid = pi;
1011                 }
1012         }
1013         return allowed_max_txq;
1014 }
1015
1016 /*
1017  * Check input txq is valid or not.
1018  * If input txq is not greater than any of maximum number
1019  * of TX queues of all ports, it is valid.
1020  * if valid, return 0, else return -1
1021  */
1022 int
1023 check_nb_txq(queueid_t txq)
1024 {
1025         queueid_t allowed_max_txq;
1026         portid_t pid = 0;
1027
1028         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1029         if (txq > allowed_max_txq) {
1030                 printf("Fail: input txq (%u) can't be greater "
1031                        "than max_tx_queues (%u) of port %u\n",
1032                        txq,
1033                        allowed_max_txq,
1034                        pid);
1035                 return -1;
1036         }
1037         return 0;
1038 }
1039
1040 static void
1041 init_config(void)
1042 {
1043         portid_t pid;
1044         struct rte_port *port;
1045         struct rte_mempool *mbp;
1046         unsigned int nb_mbuf_per_pool;
1047         lcoreid_t  lc_id;
1048         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1049         struct rte_gro_param gro_param;
1050         uint32_t gso_types;
1051         int k;
1052
1053         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1054
1055         /* Configuration of logical cores. */
1056         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1057                                 sizeof(struct fwd_lcore *) * nb_lcores,
1058                                 RTE_CACHE_LINE_SIZE);
1059         if (fwd_lcores == NULL) {
1060                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1061                                                         "failed\n", nb_lcores);
1062         }
1063         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1064                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1065                                                sizeof(struct fwd_lcore),
1066                                                RTE_CACHE_LINE_SIZE);
1067                 if (fwd_lcores[lc_id] == NULL) {
1068                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1069                                                                 "failed\n");
1070                 }
1071                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1072         }
1073
1074         RTE_ETH_FOREACH_DEV(pid) {
1075                 port = &ports[pid];
1076                 /* Apply default TxRx configuration for all ports */
1077                 port->dev_conf.txmode = tx_mode;
1078                 port->dev_conf.rxmode = rx_mode;
1079                 rte_eth_dev_info_get(pid, &port->dev_info);
1080
1081                 if (!(port->dev_info.tx_offload_capa &
1082                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1083                         port->dev_conf.txmode.offloads &=
1084                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1085                 if (!(port->dev_info.tx_offload_capa &
1086                         DEV_TX_OFFLOAD_MATCH_METADATA))
1087                         port->dev_conf.txmode.offloads &=
1088                                 ~DEV_TX_OFFLOAD_MATCH_METADATA;
1089                 if (numa_support) {
1090                         if (port_numa[pid] != NUMA_NO_CONFIG)
1091                                 port_per_socket[port_numa[pid]]++;
1092                         else {
1093                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1094
1095                                 /*
1096                                  * if socket_id is invalid,
1097                                  * set to the first available socket.
1098                                  */
1099                                 if (check_socket_id(socket_id) < 0)
1100                                         socket_id = socket_ids[0];
1101                                 port_per_socket[socket_id]++;
1102                         }
1103                 }
1104
1105                 /* Apply Rx offloads configuration */
1106                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1107                         port->rx_conf[k].offloads =
1108                                 port->dev_conf.rxmode.offloads;
1109                 /* Apply Tx offloads configuration */
1110                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1111                         port->tx_conf[k].offloads =
1112                                 port->dev_conf.txmode.offloads;
1113
1114                 /* set flag to initialize port/queue */
1115                 port->need_reconfig = 1;
1116                 port->need_reconfig_queues = 1;
1117                 port->tx_metadata = 0;
1118         }
1119
1120         /*
1121          * Create pools of mbuf.
1122          * If NUMA support is disabled, create a single pool of mbuf in
1123          * socket 0 memory by default.
1124          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1125          *
1126          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1127          * nb_txd can be configured at run time.
1128          */
1129         if (param_total_num_mbufs)
1130                 nb_mbuf_per_pool = param_total_num_mbufs;
1131         else {
1132                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1133                         (nb_lcores * mb_mempool_cache) +
1134                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1135                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1136         }
1137
1138         if (numa_support) {
1139                 uint8_t i;
1140
1141                 for (i = 0; i < num_sockets; i++)
1142                         mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool,
1143                                          socket_ids[i]);
1144         } else {
1145                 if (socket_num == UMA_NO_CONFIG)
1146                         mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 0);
1147                 else
1148                         mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool,
1149                                                  socket_num);
1150         }
1151
1152         init_port_config();
1153
1154         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1155                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1156         /*
1157          * Records which Mbuf pool to use by each logical core, if needed.
1158          */
1159         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1160                 mbp = mbuf_pool_find(
1161                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]));
1162
1163                 if (mbp == NULL)
1164                         mbp = mbuf_pool_find(0);
1165                 fwd_lcores[lc_id]->mbp = mbp;
1166                 /* initialize GSO context */
1167                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1168                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1169                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1170                 fwd_lcores[lc_id]->gso_ctx.gso_size = ETHER_MAX_LEN -
1171                         ETHER_CRC_LEN;
1172                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1173         }
1174
1175         /* Configuration of packet forwarding streams. */
1176         if (init_fwd_streams() < 0)
1177                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1178
1179         fwd_config_setup();
1180
1181         /* create a gro context for each lcore */
1182         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1183         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1184         gro_param.max_item_per_flow = MAX_PKT_BURST;
1185         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1186                 gro_param.socket_id = rte_lcore_to_socket_id(
1187                                 fwd_lcores_cpuids[lc_id]);
1188                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1189                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1190                         rte_exit(EXIT_FAILURE,
1191                                         "rte_gro_ctx_create() failed\n");
1192                 }
1193         }
1194
1195 #if defined RTE_LIBRTE_PMD_SOFTNIC
1196         if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) {
1197                 RTE_ETH_FOREACH_DEV(pid) {
1198                         port = &ports[pid];
1199                         const char *driver = port->dev_info.driver_name;
1200
1201                         if (strcmp(driver, "net_softnic") == 0)
1202                                 port->softport.fwd_lcore_arg = fwd_lcores;
1203                 }
1204         }
1205 #endif
1206
1207 }
1208
1209
1210 void
1211 reconfig(portid_t new_port_id, unsigned socket_id)
1212 {
1213         struct rte_port *port;
1214
1215         /* Reconfiguration of Ethernet ports. */
1216         port = &ports[new_port_id];
1217         rte_eth_dev_info_get(new_port_id, &port->dev_info);
1218
1219         /* set flag to initialize port/queue */
1220         port->need_reconfig = 1;
1221         port->need_reconfig_queues = 1;
1222         port->socket_id = socket_id;
1223
1224         init_port_config();
1225 }
1226
1227
1228 int
1229 init_fwd_streams(void)
1230 {
1231         portid_t pid;
1232         struct rte_port *port;
1233         streamid_t sm_id, nb_fwd_streams_new;
1234         queueid_t q;
1235
1236         /* set socket id according to numa or not */
1237         RTE_ETH_FOREACH_DEV(pid) {
1238                 port = &ports[pid];
1239                 if (nb_rxq > port->dev_info.max_rx_queues) {
1240                         printf("Fail: nb_rxq(%d) is greater than "
1241                                 "max_rx_queues(%d)\n", nb_rxq,
1242                                 port->dev_info.max_rx_queues);
1243                         return -1;
1244                 }
1245                 if (nb_txq > port->dev_info.max_tx_queues) {
1246                         printf("Fail: nb_txq(%d) is greater than "
1247                                 "max_tx_queues(%d)\n", nb_txq,
1248                                 port->dev_info.max_tx_queues);
1249                         return -1;
1250                 }
1251                 if (numa_support) {
1252                         if (port_numa[pid] != NUMA_NO_CONFIG)
1253                                 port->socket_id = port_numa[pid];
1254                         else {
1255                                 port->socket_id = rte_eth_dev_socket_id(pid);
1256
1257                                 /*
1258                                  * if socket_id is invalid,
1259                                  * set to the first available socket.
1260                                  */
1261                                 if (check_socket_id(port->socket_id) < 0)
1262                                         port->socket_id = socket_ids[0];
1263                         }
1264                 }
1265                 else {
1266                         if (socket_num == UMA_NO_CONFIG)
1267                                 port->socket_id = 0;
1268                         else
1269                                 port->socket_id = socket_num;
1270                 }
1271         }
1272
1273         q = RTE_MAX(nb_rxq, nb_txq);
1274         if (q == 0) {
1275                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1276                 return -1;
1277         }
1278         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1279         if (nb_fwd_streams_new == nb_fwd_streams)
1280                 return 0;
1281         /* clear the old */
1282         if (fwd_streams != NULL) {
1283                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1284                         if (fwd_streams[sm_id] == NULL)
1285                                 continue;
1286                         rte_free(fwd_streams[sm_id]);
1287                         fwd_streams[sm_id] = NULL;
1288                 }
1289                 rte_free(fwd_streams);
1290                 fwd_streams = NULL;
1291         }
1292
1293         /* init new */
1294         nb_fwd_streams = nb_fwd_streams_new;
1295         if (nb_fwd_streams) {
1296                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1297                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1298                         RTE_CACHE_LINE_SIZE);
1299                 if (fwd_streams == NULL)
1300                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1301                                  " (struct fwd_stream *)) failed\n",
1302                                  nb_fwd_streams);
1303
1304                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1305                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1306                                 " struct fwd_stream", sizeof(struct fwd_stream),
1307                                 RTE_CACHE_LINE_SIZE);
1308                         if (fwd_streams[sm_id] == NULL)
1309                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1310                                          "(struct fwd_stream) failed\n");
1311                 }
1312         }
1313
1314         return 0;
1315 }
1316
1317 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1318 static void
1319 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1320 {
1321         unsigned int total_burst;
1322         unsigned int nb_burst;
1323         unsigned int burst_stats[3];
1324         uint16_t pktnb_stats[3];
1325         uint16_t nb_pkt;
1326         int burst_percent[3];
1327
1328         /*
1329          * First compute the total number of packet bursts and the
1330          * two highest numbers of bursts of the same number of packets.
1331          */
1332         total_burst = 0;
1333         burst_stats[0] = burst_stats[1] = burst_stats[2] = 0;
1334         pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0;
1335         for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1336                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1337                 if (nb_burst == 0)
1338                         continue;
1339                 total_burst += nb_burst;
1340                 if (nb_burst > burst_stats[0]) {
1341                         burst_stats[1] = burst_stats[0];
1342                         pktnb_stats[1] = pktnb_stats[0];
1343                         burst_stats[0] = nb_burst;
1344                         pktnb_stats[0] = nb_pkt;
1345                 } else if (nb_burst > burst_stats[1]) {
1346                         burst_stats[1] = nb_burst;
1347                         pktnb_stats[1] = nb_pkt;
1348                 }
1349         }
1350         if (total_burst == 0)
1351                 return;
1352         burst_percent[0] = (burst_stats[0] * 100) / total_burst;
1353         printf("  %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst,
1354                burst_percent[0], (int) pktnb_stats[0]);
1355         if (burst_stats[0] == total_burst) {
1356                 printf("]\n");
1357                 return;
1358         }
1359         if (burst_stats[0] + burst_stats[1] == total_burst) {
1360                 printf(" + %d%% of %d pkts]\n",
1361                        100 - burst_percent[0], pktnb_stats[1]);
1362                 return;
1363         }
1364         burst_percent[1] = (burst_stats[1] * 100) / total_burst;
1365         burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]);
1366         if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) {
1367                 printf(" + %d%% of others]\n", 100 - burst_percent[0]);
1368                 return;
1369         }
1370         printf(" + %d%% of %d pkts + %d%% of others]\n",
1371                burst_percent[1], (int) pktnb_stats[1], burst_percent[2]);
1372 }
1373 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
1374
1375 static void
1376 fwd_port_stats_display(portid_t port_id, struct rte_eth_stats *stats)
1377 {
1378         struct rte_port *port;
1379         uint8_t i;
1380
1381         static const char *fwd_stats_border = "----------------------";
1382
1383         port = &ports[port_id];
1384         printf("\n  %s Forward statistics for port %-2d %s\n",
1385                fwd_stats_border, port_id, fwd_stats_border);
1386
1387         if ((!port->rx_queue_stats_mapping_enabled) && (!port->tx_queue_stats_mapping_enabled)) {
1388                 printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1389                        "%-"PRIu64"\n",
1390                        stats->ipackets, stats->imissed,
1391                        (uint64_t) (stats->ipackets + stats->imissed));
1392
1393                 if (cur_fwd_eng == &csum_fwd_engine)
1394                         printf("  Bad-ipcsum: %-14"PRIu64" Bad-l4csum: %-14"PRIu64"Bad-outer-l4csum: %-14"PRIu64"\n",
1395                                port->rx_bad_ip_csum, port->rx_bad_l4_csum,
1396                                port->rx_bad_outer_l4_csum);
1397                 if ((stats->ierrors + stats->rx_nombuf) > 0) {
1398                         printf("  RX-error: %-"PRIu64"\n",  stats->ierrors);
1399                         printf("  RX-nombufs: %-14"PRIu64"\n", stats->rx_nombuf);
1400                 }
1401
1402                 printf("  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1403                        "%-"PRIu64"\n",
1404                        stats->opackets, port->tx_dropped,
1405                        (uint64_t) (stats->opackets + port->tx_dropped));
1406         }
1407         else {
1408                 printf("  RX-packets:             %14"PRIu64"    RX-dropped:%14"PRIu64"    RX-total:"
1409                        "%14"PRIu64"\n",
1410                        stats->ipackets, stats->imissed,
1411                        (uint64_t) (stats->ipackets + stats->imissed));
1412
1413                 if (cur_fwd_eng == &csum_fwd_engine)
1414                         printf("  Bad-ipcsum:%14"PRIu64"    Bad-l4csum:%14"PRIu64"    Bad-outer-l4csum: %-14"PRIu64"\n",
1415                                port->rx_bad_ip_csum, port->rx_bad_l4_csum,
1416                                port->rx_bad_outer_l4_csum);
1417                 if ((stats->ierrors + stats->rx_nombuf) > 0) {
1418                         printf("  RX-error:%"PRIu64"\n", stats->ierrors);
1419                         printf("  RX-nombufs:             %14"PRIu64"\n",
1420                                stats->rx_nombuf);
1421                 }
1422
1423                 printf("  TX-packets:             %14"PRIu64"    TX-dropped:%14"PRIu64"    TX-total:"
1424                        "%14"PRIu64"\n",
1425                        stats->opackets, port->tx_dropped,
1426                        (uint64_t) (stats->opackets + port->tx_dropped));
1427         }
1428
1429 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1430         if (port->rx_stream)
1431                 pkt_burst_stats_display("RX",
1432                         &port->rx_stream->rx_burst_stats);
1433         if (port->tx_stream)
1434                 pkt_burst_stats_display("TX",
1435                         &port->tx_stream->tx_burst_stats);
1436 #endif
1437
1438         if (port->rx_queue_stats_mapping_enabled) {
1439                 printf("\n");
1440                 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) {
1441                         printf("  Stats reg %2d RX-packets:%14"PRIu64
1442                                "     RX-errors:%14"PRIu64
1443                                "    RX-bytes:%14"PRIu64"\n",
1444                                i, stats->q_ipackets[i], stats->q_errors[i], stats->q_ibytes[i]);
1445                 }
1446                 printf("\n");
1447         }
1448         if (port->tx_queue_stats_mapping_enabled) {
1449                 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) {
1450                         printf("  Stats reg %2d TX-packets:%14"PRIu64
1451                                "                                 TX-bytes:%14"PRIu64"\n",
1452                                i, stats->q_opackets[i], stats->q_obytes[i]);
1453                 }
1454         }
1455
1456         printf("  %s--------------------------------%s\n",
1457                fwd_stats_border, fwd_stats_border);
1458 }
1459
1460 static void
1461 fwd_stream_stats_display(streamid_t stream_id)
1462 {
1463         struct fwd_stream *fs;
1464         static const char *fwd_top_stats_border = "-------";
1465
1466         fs = fwd_streams[stream_id];
1467         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1468             (fs->fwd_dropped == 0))
1469                 return;
1470         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1471                "TX Port=%2d/Queue=%2d %s\n",
1472                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1473                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1474         printf("  RX-packets: %-14u TX-packets: %-14u TX-dropped: %-14u",
1475                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1476
1477         /* if checksum mode */
1478         if (cur_fwd_eng == &csum_fwd_engine) {
1479                printf("  RX- bad IP checksum: %-14u  Rx- bad L4 checksum: "
1480                         "%-14u Rx- bad outer L4 checksum: %-14u\n",
1481                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1482                         fs->rx_bad_outer_l4_csum);
1483         }
1484
1485 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1486         pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1487         pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1488 #endif
1489 }
1490
1491 static void
1492 flush_fwd_rx_queues(void)
1493 {
1494         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1495         portid_t  rxp;
1496         portid_t port_id;
1497         queueid_t rxq;
1498         uint16_t  nb_rx;
1499         uint16_t  i;
1500         uint8_t   j;
1501         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
1502         uint64_t timer_period;
1503
1504         /* convert to number of cycles */
1505         timer_period = rte_get_timer_hz(); /* 1 second timeout */
1506
1507         for (j = 0; j < 2; j++) {
1508                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
1509                         for (rxq = 0; rxq < nb_rxq; rxq++) {
1510                                 port_id = fwd_ports_ids[rxp];
1511                                 /**
1512                                 * testpmd can stuck in the below do while loop
1513                                 * if rte_eth_rx_burst() always returns nonzero
1514                                 * packets. So timer is added to exit this loop
1515                                 * after 1sec timer expiry.
1516                                 */
1517                                 prev_tsc = rte_rdtsc();
1518                                 do {
1519                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
1520                                                 pkts_burst, MAX_PKT_BURST);
1521                                         for (i = 0; i < nb_rx; i++)
1522                                                 rte_pktmbuf_free(pkts_burst[i]);
1523
1524                                         cur_tsc = rte_rdtsc();
1525                                         diff_tsc = cur_tsc - prev_tsc;
1526                                         timer_tsc += diff_tsc;
1527                                 } while ((nb_rx > 0) &&
1528                                         (timer_tsc < timer_period));
1529                                 timer_tsc = 0;
1530                         }
1531                 }
1532                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
1533         }
1534 }
1535
1536 static void
1537 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
1538 {
1539         struct fwd_stream **fsm;
1540         streamid_t nb_fs;
1541         streamid_t sm_id;
1542 #ifdef RTE_LIBRTE_BITRATE
1543         uint64_t tics_per_1sec;
1544         uint64_t tics_datum;
1545         uint64_t tics_current;
1546         uint16_t i, cnt_ports;
1547
1548         cnt_ports = nb_ports;
1549         tics_datum = rte_rdtsc();
1550         tics_per_1sec = rte_get_timer_hz();
1551 #endif
1552         fsm = &fwd_streams[fc->stream_idx];
1553         nb_fs = fc->stream_nb;
1554         do {
1555                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
1556                         (*pkt_fwd)(fsm[sm_id]);
1557 #ifdef RTE_LIBRTE_BITRATE
1558                 if (bitrate_enabled != 0 &&
1559                                 bitrate_lcore_id == rte_lcore_id()) {
1560                         tics_current = rte_rdtsc();
1561                         if (tics_current - tics_datum >= tics_per_1sec) {
1562                                 /* Periodic bitrate calculation */
1563                                 for (i = 0; i < cnt_ports; i++)
1564                                         rte_stats_bitrate_calc(bitrate_data,
1565                                                 ports_ids[i]);
1566                                 tics_datum = tics_current;
1567                         }
1568                 }
1569 #endif
1570 #ifdef RTE_LIBRTE_LATENCY_STATS
1571                 if (latencystats_enabled != 0 &&
1572                                 latencystats_lcore_id == rte_lcore_id())
1573                         rte_latencystats_update();
1574 #endif
1575
1576         } while (! fc->stopped);
1577 }
1578
1579 static int
1580 start_pkt_forward_on_core(void *fwd_arg)
1581 {
1582         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
1583                              cur_fwd_config.fwd_eng->packet_fwd);
1584         return 0;
1585 }
1586
1587 /*
1588  * Run the TXONLY packet forwarding engine to send a single burst of packets.
1589  * Used to start communication flows in network loopback test configurations.
1590  */
1591 static int
1592 run_one_txonly_burst_on_core(void *fwd_arg)
1593 {
1594         struct fwd_lcore *fwd_lc;
1595         struct fwd_lcore tmp_lcore;
1596
1597         fwd_lc = (struct fwd_lcore *) fwd_arg;
1598         tmp_lcore = *fwd_lc;
1599         tmp_lcore.stopped = 1;
1600         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
1601         return 0;
1602 }
1603
1604 /*
1605  * Launch packet forwarding:
1606  *     - Setup per-port forwarding context.
1607  *     - launch logical cores with their forwarding configuration.
1608  */
1609 static void
1610 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
1611 {
1612         port_fwd_begin_t port_fwd_begin;
1613         unsigned int i;
1614         unsigned int lc_id;
1615         int diag;
1616
1617         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
1618         if (port_fwd_begin != NULL) {
1619                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1620                         (*port_fwd_begin)(fwd_ports_ids[i]);
1621         }
1622         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
1623                 lc_id = fwd_lcores_cpuids[i];
1624                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
1625                         fwd_lcores[i]->stopped = 0;
1626                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
1627                                                      fwd_lcores[i], lc_id);
1628                         if (diag != 0)
1629                                 printf("launch lcore %u failed - diag=%d\n",
1630                                        lc_id, diag);
1631                 }
1632         }
1633 }
1634
1635 /*
1636  * Launch packet forwarding configuration.
1637  */
1638 void
1639 start_packet_forwarding(int with_tx_first)
1640 {
1641         port_fwd_begin_t port_fwd_begin;
1642         port_fwd_end_t  port_fwd_end;
1643         struct rte_port *port;
1644         unsigned int i;
1645         portid_t   pt_id;
1646         streamid_t sm_id;
1647
1648         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
1649                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
1650
1651         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
1652                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
1653
1654         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
1655                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
1656                 (!nb_rxq || !nb_txq))
1657                 rte_exit(EXIT_FAILURE,
1658                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
1659                         cur_fwd_eng->fwd_mode_name);
1660
1661         if (all_ports_started() == 0) {
1662                 printf("Not all ports were started\n");
1663                 return;
1664         }
1665         if (test_done == 0) {
1666                 printf("Packet forwarding already started\n");
1667                 return;
1668         }
1669
1670
1671         if(dcb_test) {
1672                 for (i = 0; i < nb_fwd_ports; i++) {
1673                         pt_id = fwd_ports_ids[i];
1674                         port = &ports[pt_id];
1675                         if (!port->dcb_flag) {
1676                                 printf("In DCB mode, all forwarding ports must "
1677                                        "be configured in this mode.\n");
1678                                 return;
1679                         }
1680                 }
1681                 if (nb_fwd_lcores == 1) {
1682                         printf("In DCB mode,the nb forwarding cores "
1683                                "should be larger than 1.\n");
1684                         return;
1685                 }
1686         }
1687         test_done = 0;
1688
1689         fwd_config_setup();
1690
1691         if(!no_flush_rx)
1692                 flush_fwd_rx_queues();
1693
1694         pkt_fwd_config_display(&cur_fwd_config);
1695         rxtx_config_display();
1696
1697         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1698                 pt_id = fwd_ports_ids[i];
1699                 port = &ports[pt_id];
1700                 rte_eth_stats_get(pt_id, &port->stats);
1701                 port->tx_dropped = 0;
1702
1703                 map_port_queue_stats_mapping_registers(pt_id, port);
1704         }
1705         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1706                 fwd_streams[sm_id]->rx_packets = 0;
1707                 fwd_streams[sm_id]->tx_packets = 0;
1708                 fwd_streams[sm_id]->fwd_dropped = 0;
1709                 fwd_streams[sm_id]->rx_bad_ip_csum = 0;
1710                 fwd_streams[sm_id]->rx_bad_l4_csum = 0;
1711                 fwd_streams[sm_id]->rx_bad_outer_l4_csum = 0;
1712
1713 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1714                 memset(&fwd_streams[sm_id]->rx_burst_stats, 0,
1715                        sizeof(fwd_streams[sm_id]->rx_burst_stats));
1716                 memset(&fwd_streams[sm_id]->tx_burst_stats, 0,
1717                        sizeof(fwd_streams[sm_id]->tx_burst_stats));
1718 #endif
1719 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1720                 fwd_streams[sm_id]->core_cycles = 0;
1721 #endif
1722         }
1723         if (with_tx_first) {
1724                 port_fwd_begin = tx_only_engine.port_fwd_begin;
1725                 if (port_fwd_begin != NULL) {
1726                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1727                                 (*port_fwd_begin)(fwd_ports_ids[i]);
1728                 }
1729                 while (with_tx_first--) {
1730                         launch_packet_forwarding(
1731                                         run_one_txonly_burst_on_core);
1732                         rte_eal_mp_wait_lcore();
1733                 }
1734                 port_fwd_end = tx_only_engine.port_fwd_end;
1735                 if (port_fwd_end != NULL) {
1736                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
1737                                 (*port_fwd_end)(fwd_ports_ids[i]);
1738                 }
1739         }
1740         launch_packet_forwarding(start_pkt_forward_on_core);
1741 }
1742
1743 void
1744 stop_packet_forwarding(void)
1745 {
1746         struct rte_eth_stats stats;
1747         struct rte_port *port;
1748         port_fwd_end_t  port_fwd_end;
1749         int i;
1750         portid_t   pt_id;
1751         streamid_t sm_id;
1752         lcoreid_t  lc_id;
1753         uint64_t total_recv;
1754         uint64_t total_xmit;
1755         uint64_t total_rx_dropped;
1756         uint64_t total_tx_dropped;
1757         uint64_t total_rx_nombuf;
1758         uint64_t tx_dropped;
1759         uint64_t rx_bad_ip_csum;
1760         uint64_t rx_bad_l4_csum;
1761 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1762         uint64_t fwd_cycles;
1763 #endif
1764
1765         static const char *acc_stats_border = "+++++++++++++++";
1766
1767         if (test_done) {
1768                 printf("Packet forwarding not started\n");
1769                 return;
1770         }
1771         printf("Telling cores to stop...");
1772         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
1773                 fwd_lcores[lc_id]->stopped = 1;
1774         printf("\nWaiting for lcores to finish...\n");
1775         rte_eal_mp_wait_lcore();
1776         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
1777         if (port_fwd_end != NULL) {
1778                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1779                         pt_id = fwd_ports_ids[i];
1780                         (*port_fwd_end)(pt_id);
1781                 }
1782         }
1783 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1784         fwd_cycles = 0;
1785 #endif
1786         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1787                 if (cur_fwd_config.nb_fwd_streams >
1788                     cur_fwd_config.nb_fwd_ports) {
1789                         fwd_stream_stats_display(sm_id);
1790                         ports[fwd_streams[sm_id]->tx_port].tx_stream = NULL;
1791                         ports[fwd_streams[sm_id]->rx_port].rx_stream = NULL;
1792                 } else {
1793                         ports[fwd_streams[sm_id]->tx_port].tx_stream =
1794                                 fwd_streams[sm_id];
1795                         ports[fwd_streams[sm_id]->rx_port].rx_stream =
1796                                 fwd_streams[sm_id];
1797                 }
1798                 tx_dropped = ports[fwd_streams[sm_id]->tx_port].tx_dropped;
1799                 tx_dropped = (uint64_t) (tx_dropped +
1800                                          fwd_streams[sm_id]->fwd_dropped);
1801                 ports[fwd_streams[sm_id]->tx_port].tx_dropped = tx_dropped;
1802
1803                 rx_bad_ip_csum =
1804                         ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum;
1805                 rx_bad_ip_csum = (uint64_t) (rx_bad_ip_csum +
1806                                          fwd_streams[sm_id]->rx_bad_ip_csum);
1807                 ports[fwd_streams[sm_id]->rx_port].rx_bad_ip_csum =
1808                                                         rx_bad_ip_csum;
1809
1810                 rx_bad_l4_csum =
1811                         ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum;
1812                 rx_bad_l4_csum = (uint64_t) (rx_bad_l4_csum +
1813                                          fwd_streams[sm_id]->rx_bad_l4_csum);
1814                 ports[fwd_streams[sm_id]->rx_port].rx_bad_l4_csum =
1815                                                         rx_bad_l4_csum;
1816
1817                 ports[fwd_streams[sm_id]->rx_port].rx_bad_outer_l4_csum +=
1818                                 fwd_streams[sm_id]->rx_bad_outer_l4_csum;
1819
1820 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1821                 fwd_cycles = (uint64_t) (fwd_cycles +
1822                                          fwd_streams[sm_id]->core_cycles);
1823 #endif
1824         }
1825         total_recv = 0;
1826         total_xmit = 0;
1827         total_rx_dropped = 0;
1828         total_tx_dropped = 0;
1829         total_rx_nombuf  = 0;
1830         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1831                 pt_id = fwd_ports_ids[i];
1832
1833                 port = &ports[pt_id];
1834                 rte_eth_stats_get(pt_id, &stats);
1835                 stats.ipackets -= port->stats.ipackets;
1836                 port->stats.ipackets = 0;
1837                 stats.opackets -= port->stats.opackets;
1838                 port->stats.opackets = 0;
1839                 stats.ibytes   -= port->stats.ibytes;
1840                 port->stats.ibytes = 0;
1841                 stats.obytes   -= port->stats.obytes;
1842                 port->stats.obytes = 0;
1843                 stats.imissed  -= port->stats.imissed;
1844                 port->stats.imissed = 0;
1845                 stats.oerrors  -= port->stats.oerrors;
1846                 port->stats.oerrors = 0;
1847                 stats.rx_nombuf -= port->stats.rx_nombuf;
1848                 port->stats.rx_nombuf = 0;
1849
1850                 total_recv += stats.ipackets;
1851                 total_xmit += stats.opackets;
1852                 total_rx_dropped += stats.imissed;
1853                 total_tx_dropped += port->tx_dropped;
1854                 total_rx_nombuf  += stats.rx_nombuf;
1855
1856                 fwd_port_stats_display(pt_id, &stats);
1857         }
1858
1859         printf("\n  %s Accumulated forward statistics for all ports"
1860                "%s\n",
1861                acc_stats_border, acc_stats_border);
1862         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1863                "%-"PRIu64"\n"
1864                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1865                "%-"PRIu64"\n",
1866                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1867                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1868         if (total_rx_nombuf > 0)
1869                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1870         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1871                "%s\n",
1872                acc_stats_border, acc_stats_border);
1873 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1874         if (total_recv > 0)
1875                 printf("\n  CPU cycles/packet=%u (total cycles="
1876                        "%"PRIu64" / total RX packets=%"PRIu64")\n",
1877                        (unsigned int)(fwd_cycles / total_recv),
1878                        fwd_cycles, total_recv);
1879 #endif
1880         printf("\nDone.\n");
1881         test_done = 1;
1882 }
1883
1884 void
1885 dev_set_link_up(portid_t pid)
1886 {
1887         if (rte_eth_dev_set_link_up(pid) < 0)
1888                 printf("\nSet link up fail.\n");
1889 }
1890
1891 void
1892 dev_set_link_down(portid_t pid)
1893 {
1894         if (rte_eth_dev_set_link_down(pid) < 0)
1895                 printf("\nSet link down fail.\n");
1896 }
1897
1898 static int
1899 all_ports_started(void)
1900 {
1901         portid_t pi;
1902         struct rte_port *port;
1903
1904         RTE_ETH_FOREACH_DEV(pi) {
1905                 port = &ports[pi];
1906                 /* Check if there is a port which is not started */
1907                 if ((port->port_status != RTE_PORT_STARTED) &&
1908                         (port->slave_flag == 0))
1909                         return 0;
1910         }
1911
1912         /* No port is not started */
1913         return 1;
1914 }
1915
1916 int
1917 port_is_stopped(portid_t port_id)
1918 {
1919         struct rte_port *port = &ports[port_id];
1920
1921         if ((port->port_status != RTE_PORT_STOPPED) &&
1922             (port->slave_flag == 0))
1923                 return 0;
1924         return 1;
1925 }
1926
1927 int
1928 all_ports_stopped(void)
1929 {
1930         portid_t pi;
1931
1932         RTE_ETH_FOREACH_DEV(pi) {
1933                 if (!port_is_stopped(pi))
1934                         return 0;
1935         }
1936
1937         return 1;
1938 }
1939
1940 int
1941 port_is_started(portid_t port_id)
1942 {
1943         if (port_id_is_invalid(port_id, ENABLED_WARN))
1944                 return 0;
1945
1946         if (ports[port_id].port_status != RTE_PORT_STARTED)
1947                 return 0;
1948
1949         return 1;
1950 }
1951
1952 int
1953 start_port(portid_t pid)
1954 {
1955         int diag, need_check_link_status = -1;
1956         portid_t pi;
1957         queueid_t qi;
1958         struct rte_port *port;
1959         struct ether_addr mac_addr;
1960
1961         if (port_id_is_invalid(pid, ENABLED_WARN))
1962                 return 0;
1963
1964         if(dcb_config)
1965                 dcb_test = 1;
1966         RTE_ETH_FOREACH_DEV(pi) {
1967                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
1968                         continue;
1969
1970                 need_check_link_status = 0;
1971                 port = &ports[pi];
1972                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
1973                                                  RTE_PORT_HANDLING) == 0) {
1974                         printf("Port %d is now not stopped\n", pi);
1975                         continue;
1976                 }
1977
1978                 if (port->need_reconfig > 0) {
1979                         port->need_reconfig = 0;
1980
1981                         if (flow_isolate_all) {
1982                                 int ret = port_flow_isolate(pi, 1);
1983                                 if (ret) {
1984                                         printf("Failed to apply isolated"
1985                                                " mode on port %d\n", pi);
1986                                         return -1;
1987                                 }
1988                         }
1989                         configure_rxtx_dump_callbacks(0);
1990                         printf("Configuring Port %d (socket %u)\n", pi,
1991                                         port->socket_id);
1992                         /* configure port */
1993                         diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq,
1994                                                 &(port->dev_conf));
1995                         if (diag != 0) {
1996                                 if (rte_atomic16_cmpset(&(port->port_status),
1997                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
1998                                         printf("Port %d can not be set back "
1999                                                         "to stopped\n", pi);
2000                                 printf("Fail to configure port %d\n", pi);
2001                                 /* try to reconfigure port next time */
2002                                 port->need_reconfig = 1;
2003                                 return -1;
2004                         }
2005                 }
2006                 if (port->need_reconfig_queues > 0) {
2007                         port->need_reconfig_queues = 0;
2008                         /* setup tx queues */
2009                         for (qi = 0; qi < nb_txq; qi++) {
2010                                 if ((numa_support) &&
2011                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2012                                         diag = rte_eth_tx_queue_setup(pi, qi,
2013                                                 port->nb_tx_desc[qi],
2014                                                 txring_numa[pi],
2015                                                 &(port->tx_conf[qi]));
2016                                 else
2017                                         diag = rte_eth_tx_queue_setup(pi, qi,
2018                                                 port->nb_tx_desc[qi],
2019                                                 port->socket_id,
2020                                                 &(port->tx_conf[qi]));
2021
2022                                 if (diag == 0)
2023                                         continue;
2024
2025                                 /* Fail to setup tx queue, return */
2026                                 if (rte_atomic16_cmpset(&(port->port_status),
2027                                                         RTE_PORT_HANDLING,
2028                                                         RTE_PORT_STOPPED) == 0)
2029                                         printf("Port %d can not be set back "
2030                                                         "to stopped\n", pi);
2031                                 printf("Fail to configure port %d tx queues\n",
2032                                        pi);
2033                                 /* try to reconfigure queues next time */
2034                                 port->need_reconfig_queues = 1;
2035                                 return -1;
2036                         }
2037                         for (qi = 0; qi < nb_rxq; qi++) {
2038                                 /* setup rx queues */
2039                                 if ((numa_support) &&
2040                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2041                                         struct rte_mempool * mp =
2042                                                 mbuf_pool_find(rxring_numa[pi]);
2043                                         if (mp == NULL) {
2044                                                 printf("Failed to setup RX queue:"
2045                                                         "No mempool allocation"
2046                                                         " on the socket %d\n",
2047                                                         rxring_numa[pi]);
2048                                                 return -1;
2049                                         }
2050
2051                                         diag = rte_eth_rx_queue_setup(pi, qi,
2052                                              port->nb_rx_desc[qi],
2053                                              rxring_numa[pi],
2054                                              &(port->rx_conf[qi]),
2055                                              mp);
2056                                 } else {
2057                                         struct rte_mempool *mp =
2058                                                 mbuf_pool_find(port->socket_id);
2059                                         if (mp == NULL) {
2060                                                 printf("Failed to setup RX queue:"
2061                                                         "No mempool allocation"
2062                                                         " on the socket %d\n",
2063                                                         port->socket_id);
2064                                                 return -1;
2065                                         }
2066                                         diag = rte_eth_rx_queue_setup(pi, qi,
2067                                              port->nb_rx_desc[qi],
2068                                              port->socket_id,
2069                                              &(port->rx_conf[qi]),
2070                                              mp);
2071                                 }
2072                                 if (diag == 0)
2073                                         continue;
2074
2075                                 /* Fail to setup rx queue, return */
2076                                 if (rte_atomic16_cmpset(&(port->port_status),
2077                                                         RTE_PORT_HANDLING,
2078                                                         RTE_PORT_STOPPED) == 0)
2079                                         printf("Port %d can not be set back "
2080                                                         "to stopped\n", pi);
2081                                 printf("Fail to configure port %d rx queues\n",
2082                                        pi);
2083                                 /* try to reconfigure queues next time */
2084                                 port->need_reconfig_queues = 1;
2085                                 return -1;
2086                         }
2087                 }
2088                 configure_rxtx_dump_callbacks(verbose_level);
2089                 /* start port */
2090                 if (rte_eth_dev_start(pi) < 0) {
2091                         printf("Fail to start port %d\n", pi);
2092
2093                         /* Fail to setup rx queue, return */
2094                         if (rte_atomic16_cmpset(&(port->port_status),
2095                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2096                                 printf("Port %d can not be set back to "
2097                                                         "stopped\n", pi);
2098                         continue;
2099                 }
2100
2101                 if (rte_atomic16_cmpset(&(port->port_status),
2102                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2103                         printf("Port %d can not be set into started\n", pi);
2104
2105                 rte_eth_macaddr_get(pi, &mac_addr);
2106                 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2107                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2108                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2109                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2110
2111                 /* at least one port started, need checking link status */
2112                 need_check_link_status = 1;
2113         }
2114
2115         if (need_check_link_status == 1 && !no_link_check)
2116                 check_all_ports_link_status(RTE_PORT_ALL);
2117         else if (need_check_link_status == 0)
2118                 printf("Please stop the ports first\n");
2119
2120         printf("Done\n");
2121         return 0;
2122 }
2123
2124 void
2125 stop_port(portid_t pid)
2126 {
2127         portid_t pi;
2128         struct rte_port *port;
2129         int need_check_link_status = 0;
2130
2131         if (dcb_test) {
2132                 dcb_test = 0;
2133                 dcb_config = 0;
2134         }
2135
2136         if (port_id_is_invalid(pid, ENABLED_WARN))
2137                 return;
2138
2139         printf("Stopping ports...\n");
2140
2141         RTE_ETH_FOREACH_DEV(pi) {
2142                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2143                         continue;
2144
2145                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2146                         printf("Please remove port %d from forwarding configuration.\n", pi);
2147                         continue;
2148                 }
2149
2150                 if (port_is_bonding_slave(pi)) {
2151                         printf("Please remove port %d from bonded device.\n", pi);
2152                         continue;
2153                 }
2154
2155                 port = &ports[pi];
2156                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2157                                                 RTE_PORT_HANDLING) == 0)
2158                         continue;
2159
2160                 rte_eth_dev_stop(pi);
2161
2162                 if (rte_atomic16_cmpset(&(port->port_status),
2163                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2164                         printf("Port %d can not be set into stopped\n", pi);
2165                 need_check_link_status = 1;
2166         }
2167         if (need_check_link_status && !no_link_check)
2168                 check_all_ports_link_status(RTE_PORT_ALL);
2169
2170         printf("Done\n");
2171 }
2172
2173 static void
2174 remove_invalid_ports_in(portid_t *array, portid_t *total)
2175 {
2176         portid_t i;
2177         portid_t new_total = 0;
2178
2179         for (i = 0; i < *total; i++)
2180                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2181                         array[new_total] = array[i];
2182                         new_total++;
2183                 }
2184         *total = new_total;
2185 }
2186
2187 static void
2188 remove_invalid_ports(void)
2189 {
2190         remove_invalid_ports_in(ports_ids, &nb_ports);
2191         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2192         nb_cfg_ports = nb_fwd_ports;
2193 }
2194
2195 void
2196 close_port(portid_t pid)
2197 {
2198         portid_t pi;
2199         struct rte_port *port;
2200
2201         if (port_id_is_invalid(pid, ENABLED_WARN))
2202                 return;
2203
2204         printf("Closing ports...\n");
2205
2206         RTE_ETH_FOREACH_DEV(pi) {
2207                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2208                         continue;
2209
2210                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2211                         printf("Please remove port %d from forwarding configuration.\n", pi);
2212                         continue;
2213                 }
2214
2215                 if (port_is_bonding_slave(pi)) {
2216                         printf("Please remove port %d from bonded device.\n", pi);
2217                         continue;
2218                 }
2219
2220                 port = &ports[pi];
2221                 if (rte_atomic16_cmpset(&(port->port_status),
2222                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2223                         printf("Port %d is already closed\n", pi);
2224                         continue;
2225                 }
2226
2227                 if (rte_atomic16_cmpset(&(port->port_status),
2228                         RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
2229                         printf("Port %d is now not stopped\n", pi);
2230                         continue;
2231                 }
2232
2233                 if (port->flow_list)
2234                         port_flow_flush(pi);
2235                 rte_eth_dev_close(pi);
2236
2237                 remove_invalid_ports();
2238
2239                 if (rte_atomic16_cmpset(&(port->port_status),
2240                         RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
2241                         printf("Port %d cannot be set to closed\n", pi);
2242         }
2243
2244         printf("Done\n");
2245 }
2246
2247 void
2248 reset_port(portid_t pid)
2249 {
2250         int diag;
2251         portid_t pi;
2252         struct rte_port *port;
2253
2254         if (port_id_is_invalid(pid, ENABLED_WARN))
2255                 return;
2256
2257         printf("Resetting ports...\n");
2258
2259         RTE_ETH_FOREACH_DEV(pi) {
2260                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2261                         continue;
2262
2263                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2264                         printf("Please remove port %d from forwarding "
2265                                "configuration.\n", pi);
2266                         continue;
2267                 }
2268
2269                 if (port_is_bonding_slave(pi)) {
2270                         printf("Please remove port %d from bonded device.\n",
2271                                pi);
2272                         continue;
2273                 }
2274
2275                 diag = rte_eth_dev_reset(pi);
2276                 if (diag == 0) {
2277                         port = &ports[pi];
2278                         port->need_reconfig = 1;
2279                         port->need_reconfig_queues = 1;
2280                 } else {
2281                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2282                 }
2283         }
2284
2285         printf("Done\n");
2286 }
2287
2288 void
2289 attach_port(char *identifier)
2290 {
2291         portid_t pi;
2292         struct rte_dev_iterator iterator;
2293
2294         printf("Attaching a new port...\n");
2295
2296         if (identifier == NULL) {
2297                 printf("Invalid parameters are specified\n");
2298                 return;
2299         }
2300
2301         if (rte_dev_probe(identifier) != 0) {
2302                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2303                 return;
2304         }
2305
2306         /* first attach mode: event */
2307         if (setup_on_probe_event) {
2308                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2309                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2310                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2311                                         ports[pi].need_setup != 0)
2312                                 setup_attached_port(pi);
2313                 return;
2314         }
2315
2316         /* second attach mode: iterator */
2317         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2318                 /* setup ports matching the devargs used for probing */
2319                 if (port_is_forwarding(pi))
2320                         continue; /* port was already attached before */
2321                 setup_attached_port(pi);
2322         }
2323 }
2324
2325 static void
2326 setup_attached_port(portid_t pi)
2327 {
2328         unsigned int socket_id;
2329
2330         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2331         /* if socket_id is invalid, set to the first available socket. */
2332         if (check_socket_id(socket_id) < 0)
2333                 socket_id = socket_ids[0];
2334         reconfig(pi, socket_id);
2335         rte_eth_promiscuous_enable(pi);
2336
2337         ports_ids[nb_ports++] = pi;
2338         fwd_ports_ids[nb_fwd_ports++] = pi;
2339         nb_cfg_ports = nb_fwd_ports;
2340         ports[pi].need_setup = 0;
2341         ports[pi].port_status = RTE_PORT_STOPPED;
2342
2343         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2344         printf("Done\n");
2345 }
2346
2347 void
2348 detach_port_device(portid_t port_id)
2349 {
2350         struct rte_device *dev;
2351         portid_t sibling;
2352
2353         printf("Removing a device...\n");
2354
2355         dev = rte_eth_devices[port_id].device;
2356         if (dev == NULL) {
2357                 printf("Device already removed\n");
2358                 return;
2359         }
2360
2361         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2362                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2363                         printf("Port not stopped\n");
2364                         return;
2365                 }
2366                 printf("Port was not closed\n");
2367                 if (ports[port_id].flow_list)
2368                         port_flow_flush(port_id);
2369         }
2370
2371         if (rte_dev_remove(dev) != 0) {
2372                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2373                 return;
2374         }
2375
2376         for (sibling = 0; sibling < RTE_MAX_ETHPORTS; sibling++) {
2377                 if (rte_eth_devices[sibling].device != dev)
2378                         continue;
2379                 /* reset mapping between old ports and removed device */
2380                 rte_eth_devices[sibling].device = NULL;
2381                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2382                         /* sibling ports are forced to be closed */
2383                         ports[sibling].port_status = RTE_PORT_CLOSED;
2384                         printf("Port %u is closed\n", sibling);
2385                 }
2386         }
2387
2388         remove_invalid_ports();
2389
2390         printf("Device of port %u is detached\n", port_id);
2391         printf("Now total ports is %d\n", nb_ports);
2392         printf("Done\n");
2393         return;
2394 }
2395
2396 void
2397 pmd_test_exit(void)
2398 {
2399         struct rte_device *device;
2400         portid_t pt_id;
2401         int ret;
2402
2403         if (test_done == 0)
2404                 stop_packet_forwarding();
2405
2406         if (ports != NULL) {
2407                 no_link_check = 1;
2408                 RTE_ETH_FOREACH_DEV(pt_id) {
2409                         printf("\nStopping port %d...\n", pt_id);
2410                         fflush(stdout);
2411                         stop_port(pt_id);
2412                 }
2413                 RTE_ETH_FOREACH_DEV(pt_id) {
2414                         printf("\nShutting down port %d...\n", pt_id);
2415                         fflush(stdout);
2416                         close_port(pt_id);
2417
2418                         /*
2419                          * This is a workaround to fix a virtio-user issue that
2420                          * requires to call clean-up routine to remove existing
2421                          * socket.
2422                          * This workaround valid only for testpmd, needs a fix
2423                          * valid for all applications.
2424                          * TODO: Implement proper resource cleanup
2425                          */
2426                         device = rte_eth_devices[pt_id].device;
2427                         if (device && !strcmp(device->driver->name, "net_virtio_user"))
2428                                 detach_port_device(pt_id);
2429                 }
2430         }
2431
2432         if (hot_plug) {
2433                 ret = rte_dev_event_monitor_stop();
2434                 if (ret) {
2435                         RTE_LOG(ERR, EAL,
2436                                 "fail to stop device event monitor.");
2437                         return;
2438                 }
2439
2440                 ret = rte_dev_event_callback_unregister(NULL,
2441                         dev_event_callback, NULL);
2442                 if (ret < 0) {
2443                         RTE_LOG(ERR, EAL,
2444                                 "fail to unregister device event callback.\n");
2445                         return;
2446                 }
2447
2448                 ret = rte_dev_hotplug_handle_disable();
2449                 if (ret) {
2450                         RTE_LOG(ERR, EAL,
2451                                 "fail to disable hotplug handling.\n");
2452                         return;
2453                 }
2454         }
2455
2456         printf("\nBye...\n");
2457 }
2458
2459 typedef void (*cmd_func_t)(void);
2460 struct pmd_test_command {
2461         const char *cmd_name;
2462         cmd_func_t cmd_func;
2463 };
2464
2465 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0]))
2466
2467 /* Check the link status of all ports in up to 9s, and print them finally */
2468 static void
2469 check_all_ports_link_status(uint32_t port_mask)
2470 {
2471 #define CHECK_INTERVAL 100 /* 100ms */
2472 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
2473         portid_t portid;
2474         uint8_t count, all_ports_up, print_flag = 0;
2475         struct rte_eth_link link;
2476
2477         printf("Checking link statuses...\n");
2478         fflush(stdout);
2479         for (count = 0; count <= MAX_CHECK_TIME; count++) {
2480                 all_ports_up = 1;
2481                 RTE_ETH_FOREACH_DEV(portid) {
2482                         if ((port_mask & (1 << portid)) == 0)
2483                                 continue;
2484                         memset(&link, 0, sizeof(link));
2485                         rte_eth_link_get_nowait(portid, &link);
2486                         /* print link status if flag set */
2487                         if (print_flag == 1) {
2488                                 if (link.link_status)
2489                                         printf(
2490                                         "Port%d Link Up. speed %u Mbps- %s\n",
2491                                         portid, link.link_speed,
2492                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
2493                                         ("full-duplex") : ("half-duplex\n"));
2494                                 else
2495                                         printf("Port %d Link Down\n", portid);
2496                                 continue;
2497                         }
2498                         /* clear all_ports_up flag if any link down */
2499                         if (link.link_status == ETH_LINK_DOWN) {
2500                                 all_ports_up = 0;
2501                                 break;
2502                         }
2503                 }
2504                 /* after finally printing all link status, get out */
2505                 if (print_flag == 1)
2506                         break;
2507
2508                 if (all_ports_up == 0) {
2509                         fflush(stdout);
2510                         rte_delay_ms(CHECK_INTERVAL);
2511                 }
2512
2513                 /* set the print_flag if all ports up or timeout */
2514                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
2515                         print_flag = 1;
2516                 }
2517
2518                 if (lsc_interrupt)
2519                         break;
2520         }
2521 }
2522
2523 /*
2524  * This callback is for remove a port for a device. It has limitation because
2525  * it is not for multiple port removal for a device.
2526  * TODO: the device detach invoke will plan to be removed from user side to
2527  * eal. And convert all PMDs to free port resources on ether device closing.
2528  */
2529 static void
2530 rmv_port_callback(void *arg)
2531 {
2532         int need_to_start = 0;
2533         int org_no_link_check = no_link_check;
2534         portid_t port_id = (intptr_t)arg;
2535
2536         RTE_ETH_VALID_PORTID_OR_RET(port_id);
2537
2538         if (!test_done && port_is_forwarding(port_id)) {
2539                 need_to_start = 1;
2540                 stop_packet_forwarding();
2541         }
2542         no_link_check = 1;
2543         stop_port(port_id);
2544         no_link_check = org_no_link_check;
2545         close_port(port_id);
2546         detach_port_device(port_id);
2547         if (need_to_start)
2548                 start_packet_forwarding(0);
2549 }
2550
2551 /* This function is used by the interrupt thread */
2552 static int
2553 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
2554                   void *ret_param)
2555 {
2556         RTE_SET_USED(param);
2557         RTE_SET_USED(ret_param);
2558
2559         if (type >= RTE_ETH_EVENT_MAX) {
2560                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
2561                         port_id, __func__, type);
2562                 fflush(stderr);
2563         } else if (event_print_mask & (UINT32_C(1) << type)) {
2564                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
2565                         eth_event_desc[type]);
2566                 fflush(stdout);
2567         }
2568
2569         switch (type) {
2570         case RTE_ETH_EVENT_NEW:
2571                 ports[port_id].need_setup = 1;
2572                 ports[port_id].port_status = RTE_PORT_HANDLING;
2573                 break;
2574         case RTE_ETH_EVENT_INTR_RMV:
2575                 if (port_id_is_invalid(port_id, DISABLED_WARN))
2576                         break;
2577                 if (rte_eal_alarm_set(100000,
2578                                 rmv_port_callback, (void *)(intptr_t)port_id))
2579                         fprintf(stderr, "Could not set up deferred device removal\n");
2580                 break;
2581         default:
2582                 break;
2583         }
2584         return 0;
2585 }
2586
2587 static int
2588 register_eth_event_callback(void)
2589 {
2590         int ret;
2591         enum rte_eth_event_type event;
2592
2593         for (event = RTE_ETH_EVENT_UNKNOWN;
2594                         event < RTE_ETH_EVENT_MAX; event++) {
2595                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
2596                                 event,
2597                                 eth_event_callback,
2598                                 NULL);
2599                 if (ret != 0) {
2600                         TESTPMD_LOG(ERR, "Failed to register callback for "
2601                                         "%s event\n", eth_event_desc[event]);
2602                         return -1;
2603                 }
2604         }
2605
2606         return 0;
2607 }
2608
2609 /* This function is used by the interrupt thread */
2610 static void
2611 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
2612                              __rte_unused void *arg)
2613 {
2614         uint16_t port_id;
2615         int ret;
2616
2617         if (type >= RTE_DEV_EVENT_MAX) {
2618                 fprintf(stderr, "%s called upon invalid event %d\n",
2619                         __func__, type);
2620                 fflush(stderr);
2621         }
2622
2623         switch (type) {
2624         case RTE_DEV_EVENT_REMOVE:
2625                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
2626                         device_name);
2627                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
2628                 if (ret) {
2629                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
2630                                 device_name);
2631                         return;
2632                 }
2633                 /*
2634                  * Because the user's callback is invoked in eal interrupt
2635                  * callback, the interrupt callback need to be finished before
2636                  * it can be unregistered when detaching device. So finish
2637                  * callback soon and use a deferred removal to detach device
2638                  * is need. It is a workaround, once the device detaching be
2639                  * moved into the eal in the future, the deferred removal could
2640                  * be deleted.
2641                  */
2642                 if (rte_eal_alarm_set(100000,
2643                                 rmv_port_callback, (void *)(intptr_t)port_id))
2644                         RTE_LOG(ERR, EAL,
2645                                 "Could not set up deferred device removal\n");
2646                 break;
2647         case RTE_DEV_EVENT_ADD:
2648                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
2649                         device_name);
2650                 /* TODO: After finish kernel driver binding,
2651                  * begin to attach port.
2652                  */
2653                 break;
2654         default:
2655                 break;
2656         }
2657 }
2658
2659 static int
2660 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2661 {
2662         uint16_t i;
2663         int diag;
2664         uint8_t mapping_found = 0;
2665
2666         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
2667                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
2668                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
2669                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
2670                                         tx_queue_stats_mappings[i].queue_id,
2671                                         tx_queue_stats_mappings[i].stats_counter_id);
2672                         if (diag != 0)
2673                                 return diag;
2674                         mapping_found = 1;
2675                 }
2676         }
2677         if (mapping_found)
2678                 port->tx_queue_stats_mapping_enabled = 1;
2679         return 0;
2680 }
2681
2682 static int
2683 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
2684 {
2685         uint16_t i;
2686         int diag;
2687         uint8_t mapping_found = 0;
2688
2689         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
2690                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
2691                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
2692                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
2693                                         rx_queue_stats_mappings[i].queue_id,
2694                                         rx_queue_stats_mappings[i].stats_counter_id);
2695                         if (diag != 0)
2696                                 return diag;
2697                         mapping_found = 1;
2698                 }
2699         }
2700         if (mapping_found)
2701                 port->rx_queue_stats_mapping_enabled = 1;
2702         return 0;
2703 }
2704
2705 static void
2706 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
2707 {
2708         int diag = 0;
2709
2710         diag = set_tx_queue_stats_mapping_registers(pi, port);
2711         if (diag != 0) {
2712                 if (diag == -ENOTSUP) {
2713                         port->tx_queue_stats_mapping_enabled = 0;
2714                         printf("TX queue stats mapping not supported port id=%d\n", pi);
2715                 }
2716                 else
2717                         rte_exit(EXIT_FAILURE,
2718                                         "set_tx_queue_stats_mapping_registers "
2719                                         "failed for port id=%d diag=%d\n",
2720                                         pi, diag);
2721         }
2722
2723         diag = set_rx_queue_stats_mapping_registers(pi, port);
2724         if (diag != 0) {
2725                 if (diag == -ENOTSUP) {
2726                         port->rx_queue_stats_mapping_enabled = 0;
2727                         printf("RX queue stats mapping not supported port id=%d\n", pi);
2728                 }
2729                 else
2730                         rte_exit(EXIT_FAILURE,
2731                                         "set_rx_queue_stats_mapping_registers "
2732                                         "failed for port id=%d diag=%d\n",
2733                                         pi, diag);
2734         }
2735 }
2736
2737 static void
2738 rxtx_port_config(struct rte_port *port)
2739 {
2740         uint16_t qid;
2741
2742         for (qid = 0; qid < nb_rxq; qid++) {
2743                 port->rx_conf[qid] = port->dev_info.default_rxconf;
2744
2745                 /* Check if any Rx parameters have been passed */
2746                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
2747                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
2748
2749                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
2750                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
2751
2752                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
2753                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
2754
2755                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
2756                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
2757
2758                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
2759                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
2760
2761                 port->nb_rx_desc[qid] = nb_rxd;
2762         }
2763
2764         for (qid = 0; qid < nb_txq; qid++) {
2765                 port->tx_conf[qid] = port->dev_info.default_txconf;
2766
2767                 /* Check if any Tx parameters have been passed */
2768                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
2769                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
2770
2771                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
2772                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
2773
2774                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
2775                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
2776
2777                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
2778                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
2779
2780                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
2781                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
2782
2783                 port->nb_tx_desc[qid] = nb_txd;
2784         }
2785 }
2786
2787 void
2788 init_port_config(void)
2789 {
2790         portid_t pid;
2791         struct rte_port *port;
2792
2793         RTE_ETH_FOREACH_DEV(pid) {
2794                 port = &ports[pid];
2795                 port->dev_conf.fdir_conf = fdir_conf;
2796                 rte_eth_dev_info_get(pid, &port->dev_info);
2797                 if (nb_rxq > 1) {
2798                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2799                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
2800                                 rss_hf & port->dev_info.flow_type_rss_offloads;
2801                 } else {
2802                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
2803                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
2804                 }
2805
2806                 if (port->dcb_flag == 0) {
2807                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
2808                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
2809                         else
2810                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
2811                 }
2812
2813                 rxtx_port_config(port);
2814
2815                 rte_eth_macaddr_get(pid, &port->eth_addr);
2816
2817                 map_port_queue_stats_mapping_registers(pid, port);
2818 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
2819                 rte_pmd_ixgbe_bypass_init(pid);
2820 #endif
2821
2822                 if (lsc_interrupt &&
2823                     (rte_eth_devices[pid].data->dev_flags &
2824                      RTE_ETH_DEV_INTR_LSC))
2825                         port->dev_conf.intr_conf.lsc = 1;
2826                 if (rmv_interrupt &&
2827                     (rte_eth_devices[pid].data->dev_flags &
2828                      RTE_ETH_DEV_INTR_RMV))
2829                         port->dev_conf.intr_conf.rmv = 1;
2830         }
2831 }
2832
2833 void set_port_slave_flag(portid_t slave_pid)
2834 {
2835         struct rte_port *port;
2836
2837         port = &ports[slave_pid];
2838         port->slave_flag = 1;
2839 }
2840
2841 void clear_port_slave_flag(portid_t slave_pid)
2842 {
2843         struct rte_port *port;
2844
2845         port = &ports[slave_pid];
2846         port->slave_flag = 0;
2847 }
2848
2849 uint8_t port_is_bonding_slave(portid_t slave_pid)
2850 {
2851         struct rte_port *port;
2852
2853         port = &ports[slave_pid];
2854         if ((rte_eth_devices[slave_pid].data->dev_flags &
2855             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
2856                 return 1;
2857         return 0;
2858 }
2859
2860 const uint16_t vlan_tags[] = {
2861                 0,  1,  2,  3,  4,  5,  6,  7,
2862                 8,  9, 10, 11,  12, 13, 14, 15,
2863                 16, 17, 18, 19, 20, 21, 22, 23,
2864                 24, 25, 26, 27, 28, 29, 30, 31
2865 };
2866
2867 static  int
2868 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
2869                  enum dcb_mode_enable dcb_mode,
2870                  enum rte_eth_nb_tcs num_tcs,
2871                  uint8_t pfc_en)
2872 {
2873         uint8_t i;
2874         int32_t rc;
2875         struct rte_eth_rss_conf rss_conf;
2876
2877         /*
2878          * Builds up the correct configuration for dcb+vt based on the vlan tags array
2879          * given above, and the number of traffic classes available for use.
2880          */
2881         if (dcb_mode == DCB_VT_ENABLED) {
2882                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
2883                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
2884                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
2885                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
2886
2887                 /* VMDQ+DCB RX and TX configurations */
2888                 vmdq_rx_conf->enable_default_pool = 0;
2889                 vmdq_rx_conf->default_pool = 0;
2890                 vmdq_rx_conf->nb_queue_pools =
2891                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
2892                 vmdq_tx_conf->nb_queue_pools =
2893                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
2894
2895                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
2896                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
2897                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
2898                         vmdq_rx_conf->pool_map[i].pools =
2899                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
2900                 }
2901                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
2902                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
2903                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
2904                 }
2905
2906                 /* set DCB mode of RX and TX of multiple queues */
2907                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB;
2908                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
2909         } else {
2910                 struct rte_eth_dcb_rx_conf *rx_conf =
2911                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
2912                 struct rte_eth_dcb_tx_conf *tx_conf =
2913                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
2914
2915                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
2916                 if (rc != 0)
2917                         return rc;
2918
2919                 rx_conf->nb_tcs = num_tcs;
2920                 tx_conf->nb_tcs = num_tcs;
2921
2922                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
2923                         rx_conf->dcb_tc[i] = i % num_tcs;
2924                         tx_conf->dcb_tc[i] = i % num_tcs;
2925                 }
2926
2927                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS;
2928                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
2929                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
2930         }
2931
2932         if (pfc_en)
2933                 eth_conf->dcb_capability_en =
2934                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
2935         else
2936                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
2937
2938         return 0;
2939 }
2940
2941 int
2942 init_port_dcb_config(portid_t pid,
2943                      enum dcb_mode_enable dcb_mode,
2944                      enum rte_eth_nb_tcs num_tcs,
2945                      uint8_t pfc_en)
2946 {
2947         struct rte_eth_conf port_conf;
2948         struct rte_port *rte_port;
2949         int retval;
2950         uint16_t i;
2951
2952         rte_port = &ports[pid];
2953
2954         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
2955         /* Enter DCB configuration status */
2956         dcb_config = 1;
2957
2958         port_conf.rxmode = rte_port->dev_conf.rxmode;
2959         port_conf.txmode = rte_port->dev_conf.txmode;
2960
2961         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
2962         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
2963         if (retval < 0)
2964                 return retval;
2965         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
2966
2967         /* re-configure the device . */
2968         rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
2969
2970         rte_eth_dev_info_get(pid, &rte_port->dev_info);
2971
2972         /* If dev_info.vmdq_pool_base is greater than 0,
2973          * the queue id of vmdq pools is started after pf queues.
2974          */
2975         if (dcb_mode == DCB_VT_ENABLED &&
2976             rte_port->dev_info.vmdq_pool_base > 0) {
2977                 printf("VMDQ_DCB multi-queue mode is nonsensical"
2978                         " for port %d.", pid);
2979                 return -1;
2980         }
2981
2982         /* Assume the ports in testpmd have the same dcb capability
2983          * and has the same number of rxq and txq in dcb mode
2984          */
2985         if (dcb_mode == DCB_VT_ENABLED) {
2986                 if (rte_port->dev_info.max_vfs > 0) {
2987                         nb_rxq = rte_port->dev_info.nb_rx_queues;
2988                         nb_txq = rte_port->dev_info.nb_tx_queues;
2989                 } else {
2990                         nb_rxq = rte_port->dev_info.max_rx_queues;
2991                         nb_txq = rte_port->dev_info.max_tx_queues;
2992                 }
2993         } else {
2994                 /*if vt is disabled, use all pf queues */
2995                 if (rte_port->dev_info.vmdq_pool_base == 0) {
2996                         nb_rxq = rte_port->dev_info.max_rx_queues;
2997                         nb_txq = rte_port->dev_info.max_tx_queues;
2998                 } else {
2999                         nb_rxq = (queueid_t)num_tcs;
3000                         nb_txq = (queueid_t)num_tcs;
3001
3002                 }
3003         }
3004         rx_free_thresh = 64;
3005
3006         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3007
3008         rxtx_port_config(rte_port);
3009         /* VLAN filter */
3010         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3011         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3012                 rx_vft_set(pid, vlan_tags[i], 1);
3013
3014         rte_eth_macaddr_get(pid, &rte_port->eth_addr);
3015         map_port_queue_stats_mapping_registers(pid, rte_port);
3016
3017         rte_port->dcb_flag = 1;
3018
3019         return 0;
3020 }
3021
3022 static void
3023 init_port(void)
3024 {
3025         /* Configuration of Ethernet ports. */
3026         ports = rte_zmalloc("testpmd: ports",
3027                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3028                             RTE_CACHE_LINE_SIZE);
3029         if (ports == NULL) {
3030                 rte_exit(EXIT_FAILURE,
3031                                 "rte_zmalloc(%d struct rte_port) failed\n",
3032                                 RTE_MAX_ETHPORTS);
3033         }
3034
3035         /* Initialize ports NUMA structures */
3036         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3037         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3038         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3039 }
3040
3041 static void
3042 force_quit(void)
3043 {
3044         pmd_test_exit();
3045         prompt_exit();
3046 }
3047
3048 static void
3049 print_stats(void)
3050 {
3051         uint8_t i;
3052         const char clr[] = { 27, '[', '2', 'J', '\0' };
3053         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3054
3055         /* Clear screen and move to top left */
3056         printf("%s%s", clr, top_left);
3057
3058         printf("\nPort statistics ====================================");
3059         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3060                 nic_stats_display(fwd_ports_ids[i]);
3061 }
3062
3063 static void
3064 signal_handler(int signum)
3065 {
3066         if (signum == SIGINT || signum == SIGTERM) {
3067                 printf("\nSignal %d received, preparing to exit...\n",
3068                                 signum);
3069 #ifdef RTE_LIBRTE_PDUMP
3070                 /* uninitialize packet capture framework */
3071                 rte_pdump_uninit();
3072 #endif
3073 #ifdef RTE_LIBRTE_LATENCY_STATS
3074                 rte_latencystats_uninit();
3075 #endif
3076                 force_quit();
3077                 /* Set flag to indicate the force termination. */
3078                 f_quit = 1;
3079                 /* exit with the expected status */
3080                 signal(signum, SIG_DFL);
3081                 kill(getpid(), signum);
3082         }
3083 }
3084
3085 int
3086 main(int argc, char** argv)
3087 {
3088         int diag;
3089         portid_t port_id;
3090         uint16_t count;
3091         int ret;
3092
3093         signal(SIGINT, signal_handler);
3094         signal(SIGTERM, signal_handler);
3095
3096         diag = rte_eal_init(argc, argv);
3097         if (diag < 0)
3098                 rte_panic("Cannot init EAL\n");
3099
3100         testpmd_logtype = rte_log_register("testpmd");
3101         if (testpmd_logtype < 0)
3102                 rte_panic("Cannot register log type");
3103         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3104
3105         ret = register_eth_event_callback();
3106         if (ret != 0)
3107                 rte_panic("Cannot register for ethdev events");
3108
3109 #ifdef RTE_LIBRTE_PDUMP
3110         /* initialize packet capture framework */
3111         rte_pdump_init(NULL);
3112 #endif
3113
3114         count = 0;
3115         RTE_ETH_FOREACH_DEV(port_id) {
3116                 ports_ids[count] = port_id;
3117                 count++;
3118         }
3119         nb_ports = (portid_t) count;
3120         if (nb_ports == 0)
3121                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3122
3123         /* allocate port structures, and init them */
3124         init_port();
3125
3126         set_def_fwd_config();
3127         if (nb_lcores == 0)
3128                 rte_panic("Empty set of forwarding logical cores - check the "
3129                           "core mask supplied in the command parameters\n");
3130
3131         /* Bitrate/latency stats disabled by default */
3132 #ifdef RTE_LIBRTE_BITRATE
3133         bitrate_enabled = 0;
3134 #endif
3135 #ifdef RTE_LIBRTE_LATENCY_STATS
3136         latencystats_enabled = 0;
3137 #endif
3138
3139         /* on FreeBSD, mlockall() is disabled by default */
3140 #ifdef RTE_EXEC_ENV_BSDAPP
3141         do_mlockall = 0;
3142 #else
3143         do_mlockall = 1;
3144 #endif
3145
3146         argc -= diag;
3147         argv += diag;
3148         if (argc > 1)
3149                 launch_args_parse(argc, argv);
3150
3151         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3152                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3153                         strerror(errno));
3154         }
3155
3156         if (tx_first && interactive)
3157                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3158                                 "interactive mode.\n");
3159
3160         if (tx_first && lsc_interrupt) {
3161                 printf("Warning: lsc_interrupt needs to be off when "
3162                                 " using tx_first. Disabling.\n");
3163                 lsc_interrupt = 0;
3164         }
3165
3166         if (!nb_rxq && !nb_txq)
3167                 printf("Warning: Either rx or tx queues should be non-zero\n");
3168
3169         if (nb_rxq > 1 && nb_rxq > nb_txq)
3170                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3171                        "but nb_txq=%d will prevent to fully test it.\n",
3172                        nb_rxq, nb_txq);
3173
3174         init_config();
3175
3176         if (hot_plug) {
3177                 ret = rte_dev_hotplug_handle_enable();
3178                 if (ret) {
3179                         RTE_LOG(ERR, EAL,
3180                                 "fail to enable hotplug handling.");
3181                         return -1;
3182                 }
3183
3184                 ret = rte_dev_event_monitor_start();
3185                 if (ret) {
3186                         RTE_LOG(ERR, EAL,
3187                                 "fail to start device event monitoring.");
3188                         return -1;
3189                 }
3190
3191                 ret = rte_dev_event_callback_register(NULL,
3192                         dev_event_callback, NULL);
3193                 if (ret) {
3194                         RTE_LOG(ERR, EAL,
3195                                 "fail  to register device event callback\n");
3196                         return -1;
3197                 }
3198         }
3199
3200         if (start_port(RTE_PORT_ALL) != 0)
3201                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3202
3203         /* set all ports to promiscuous mode by default */
3204         RTE_ETH_FOREACH_DEV(port_id)
3205                 rte_eth_promiscuous_enable(port_id);
3206
3207         /* Init metrics library */
3208         rte_metrics_init(rte_socket_id());
3209
3210 #ifdef RTE_LIBRTE_LATENCY_STATS
3211         if (latencystats_enabled != 0) {
3212                 int ret = rte_latencystats_init(1, NULL);
3213                 if (ret)
3214                         printf("Warning: latencystats init()"
3215                                 " returned error %d\n", ret);
3216                 printf("Latencystats running on lcore %d\n",
3217                         latencystats_lcore_id);
3218         }
3219 #endif
3220
3221         /* Setup bitrate stats */
3222 #ifdef RTE_LIBRTE_BITRATE
3223         if (bitrate_enabled != 0) {
3224                 bitrate_data = rte_stats_bitrate_create();
3225                 if (bitrate_data == NULL)
3226                         rte_exit(EXIT_FAILURE,
3227                                 "Could not allocate bitrate data.\n");
3228                 rte_stats_bitrate_reg(bitrate_data);
3229         }
3230 #endif
3231
3232 #ifdef RTE_LIBRTE_CMDLINE
3233         if (strlen(cmdline_filename) != 0)
3234                 cmdline_read_from_file(cmdline_filename);
3235
3236         if (interactive == 1) {
3237                 if (auto_start) {
3238                         printf("Start automatic packet forwarding\n");
3239                         start_packet_forwarding(0);
3240                 }
3241                 prompt();
3242                 pmd_test_exit();
3243         } else
3244 #endif
3245         {
3246                 char c;
3247                 int rc;
3248
3249                 f_quit = 0;
3250
3251                 printf("No commandline core given, start packet forwarding\n");
3252                 start_packet_forwarding(tx_first);
3253                 if (stats_period != 0) {
3254                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3255                         uint64_t timer_period;
3256
3257                         /* Convert to number of cycles */
3258                         timer_period = stats_period * rte_get_timer_hz();
3259
3260                         while (f_quit == 0) {
3261                                 cur_time = rte_get_timer_cycles();
3262                                 diff_time += cur_time - prev_time;
3263
3264                                 if (diff_time >= timer_period) {
3265                                         print_stats();
3266                                         /* Reset the timer */
3267                                         diff_time = 0;
3268                                 }
3269                                 /* Sleep to avoid unnecessary checks */
3270                                 prev_time = cur_time;
3271                                 sleep(1);
3272                         }
3273                 }
3274
3275                 printf("Press enter to exit\n");
3276                 rc = read(0, &c, 1);
3277                 pmd_test_exit();
3278                 if (rc < 0)
3279                         return 1;
3280         }
3281
3282         return 0;
3283 }