examples/bbdev_app/main.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <stdlib.h>
   7 #include <string.h>
   8 #include <stdint.h>
   9 #include <inttypes.h>
  10 #include <sys/types.h>
  11 #include <sys/unistd.h>
  12 #include <sys/queue.h>
  13 #include <stdarg.h>
  14 #include <ctype.h>
  15 #include <errno.h>
  16 #include <math.h>
  17 #include <assert.h>
  18 #include <getopt.h>
  19 #include <signal.h>
  20
  21 #include "rte_atomic.h"
  22 #include "rte_common.h"
  23 #include "rte_eal.h"
  24 #include "rte_cycles.h"
  25 #include "rte_ether.h"
  26 #include "rte_ethdev.h"
  27 #include "rte_ip.h"
  28 #include "rte_lcore.h"
  29 #include "rte_malloc.h"
  30 #include "rte_mbuf.h"
  31 #include "rte_memory.h"
  32 #include "rte_mempool.h"
  33 #include "rte_log.h"
  34 #include "rte_bbdev.h"
  35 #include "rte_bbdev_op.h"
  36
  37 /* LLR values - negative value for '1' bit */
  38 #define LLR_1_BIT 0x81
  39 #define LLR_0_BIT 0x7F
  40
  41 #define MAX_PKT_BURST 32
  42 #define NB_MBUF 8191
  43 #define MEMPOOL_CACHE_SIZE 256
  44
  45 /* Hardcoded K value */
  46 #define K 40
  47 #define NCB (3 * RTE_ALIGN_CEIL(K + 4, 32))
  48
  49 #define CRC_24B_LEN 3
  50
  51 /* Configurable number of RX/TX ring descriptors */
  52 #define RTE_TEST_RX_DESC_DEFAULT 128
  53 #define RTE_TEST_TX_DESC_DEFAULT 512
  54
  55 #define BBDEV_ASSERT(a) do { \
  56         if (!(a)) { \
  57                 usage(prgname); \
  58                 return -1; \
  59         } \
  60 } while (0)
  61
  62 static const struct rte_eth_conf port_conf = {
  63         .rxmode = {
  64                 .mq_mode = ETH_MQ_RX_NONE,
  65                 .max_rx_pkt_len = ETHER_MAX_LEN,
  66                 .split_hdr_size = 0,
  67                 .header_split = 0, /**< Header Split disabled */
  68                 .hw_ip_checksum = 0, /**< IP checksum offload disabled */
  69                 .hw_vlan_filter = 0, /**< VLAN filtering disabled */
  70                 .jumbo_frame = 0, /**< Jumbo Frame Support disabled */
  71                 .hw_strip_crc = 0, /**< CRC stripped by hardware */
  72         },
  73         .txmode = {
  74                 .mq_mode = ETH_MQ_TX_NONE,
  75         },
  76 };
  77
  78 struct rte_bbdev_op_turbo_enc def_op_enc = {
  79         /* These values are arbitrarily put, and does not map to the real
  80          * values for the data received from ethdev ports
  81          */
  82         .rv_index = 0,
  83         .code_block_mode = 1,
  84         .cb_params = {
  85                 .k = K,
  86         },
  87         .op_flags = RTE_BBDEV_TURBO_CRC_24A_ATTACH
  88 };
  89
  90 struct rte_bbdev_op_turbo_dec def_op_dec = {
  91         /* These values are arbitrarily put, and does not map to the real
  92          * values for the data received from ethdev ports
  93          */
  94         .code_block_mode = 1,
  95         .cb_params = {
  96                 .k = K,
  97         },
  98         .rv_index = 0,
  99         .iter_max = 8,
 100         .iter_min = 4,
 101         .ext_scale = 15,
 102         .num_maps = 0,
 103         .op_flags = RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN
 104 };
 105
 106 struct app_config_params {
 107         /* Placeholders for app params */
 108         uint16_t port_id;
 109         uint16_t bbdev_id;
 110         uint64_t enc_core_mask;
 111         uint64_t dec_core_mask;
 112
 113         /* Values filled during init time */
 114         uint16_t enc_queue_ids[RTE_MAX_LCORE];
 115         uint16_t dec_queue_ids[RTE_MAX_LCORE];
 116         uint16_t num_enc_cores;
 117         uint16_t num_dec_cores;
 118 };
 119
 120 struct lcore_statistics {
 121         unsigned int enqueued;
 122         unsigned int dequeued;
 123         unsigned int rx_lost_packets;
 124         unsigned int enc_to_dec_lost_packets;
 125         unsigned int tx_lost_packets;
 126 } __rte_cache_aligned;
 127
 128 /** each lcore configuration */
 129 struct lcore_conf {
 130         uint64_t core_type;
 131
 132         unsigned int port_id;
 133         unsigned int rx_queue_id;
 134         unsigned int tx_queue_id;
 135
 136         unsigned int bbdev_id;
 137         unsigned int enc_queue_id;
 138         unsigned int dec_queue_id;
 139
 140         uint8_t llr_temp_buf[NCB];
 141
 142         struct rte_mempool *bbdev_dec_op_pool;
 143         struct rte_mempool *bbdev_enc_op_pool;
 144         struct rte_mempool *enc_out_pool;
 145         struct rte_ring *enc_to_dec_ring;
 146
 147         struct lcore_statistics *lcore_stats;
 148 } __rte_cache_aligned;
 149
 150 struct stats_lcore_params {
 151         struct lcore_conf *lconf;
 152         struct app_config_params *app_params;
 153 };
 154
 155
 156 static const struct app_config_params def_app_config = {
 157         .port_id = 0,
 158         .bbdev_id = 0,
 159         .enc_core_mask = 0x2,
 160         .dec_core_mask = 0x4,
 161         .num_enc_cores = 1,
 162         .num_dec_cores = 1,
 163 };
 164
 165 static rte_atomic16_t global_exit_flag;
 166
 167 /* display usage */
 168 static inline void
 169 usage(const char *prgname)
 170 {
 171         printf("%s [EAL options] "
 172                         "  --\n"
 173                         "  --enc_cores - number of encoding cores (default = 0x2)\n"
 174                         "  --dec_cores - number of decoding cores (default = 0x4)\n"
 175                         "  --port_id - Ethernet port ID (default = 0)\n"
 176                         "  --bbdev_id - BBDev ID (default = 0)\n"
 177                         "\n", prgname);
 178 }
 179
 180 /* parse core mask */
 181 static inline
 182 uint16_t bbdev_parse_mask(const char *mask)
 183 {
 184         char *end = NULL;
 185         unsigned long pm;
 186
 187         /* parse hexadecimal string */
 188         pm = strtoul(mask, &end, 16);
 189         if ((mask[0] == '\0') || (end == NULL) || (*end != '\0'))
 190                 return 0;
 191
 192         return pm;
 193 }
 194
 195 /* parse core mask */
 196 static inline
 197 uint16_t bbdev_parse_number(const char *mask)
 198 {
 199         char *end = NULL;
 200         unsigned long pm;
 201
 202         /* parse hexadecimal string */
 203         pm = strtoul(mask, &end, 10);
 204         if ((mask[0] == '\0') || (end == NULL) || (*end != '\0'))
 205                 return 0;
 206
 207         return pm;
 208 }
 209
 210 static int
 211 bbdev_parse_args(int argc, char **argv,
 212                 struct app_config_params *app_params)
 213 {
 214         int optind = 0;
 215         int opt;
 216         int opt_indx = 0;
 217         char *prgname = argv[0];
 218
 219         static struct option lgopts[] = {
 220                 { "enc_core_mask", required_argument, 0, 'e' },
 221                 { "dec_core_mask", required_argument, 0, 'd' },
 222                 { "port_id", required_argument, 0, 'p' },
 223                 { "bbdev_id", required_argument, 0, 'b' },
 224                 { NULL, 0, 0, 0 }
 225         };
 226
 227         BBDEV_ASSERT(argc != 0);
 228         BBDEV_ASSERT(argv != NULL);
 229         BBDEV_ASSERT(app_params != NULL);
 230
 231         while ((opt = getopt_long(argc, argv, "e:d:p:b:", lgopts, &opt_indx)) !=
 232                 EOF) {
 233                 switch (opt) {
 234                 case 'e':
 235                         app_params->enc_core_mask =
 236                                 bbdev_parse_mask(optarg);
 237                         if (app_params->enc_core_mask == 0) {
 238                                 usage(prgname);
 239                                 return -1;
 240                         }
 241                         app_params->num_enc_cores =
 242                                 __builtin_popcount(app_params->enc_core_mask);
 243                         break;
 244
 245                 case 'd':
 246                         app_params->dec_core_mask =
 247                                 bbdev_parse_mask(optarg);
 248                         if (app_params->dec_core_mask == 0) {
 249                                 usage(prgname);
 250                                 return -1;
 251                         }
 252                         app_params->num_dec_cores =
 253                                 __builtin_popcount(app_params->dec_core_mask);
 254                         break;
 255
 256                 case 'p':
 257                         app_params->port_id = bbdev_parse_number(optarg);
 258                         break;
 259
 260                 case 'b':
 261                         app_params->bbdev_id = bbdev_parse_number(optarg);
 262                         break;
 263
 264                 default:
 265                         usage(prgname);
 266                         return -1;
 267                 }
 268         }
 269         optind = 0;
 270         return optind;
 271 }
 272
 273 static void
 274 signal_handler(int signum)
 275 {
 276         printf("\nSignal %d received\n", signum);
 277         rte_atomic16_set(&global_exit_flag, 1);
 278 }
 279
 280 static void
 281 print_mac(unsigned int portid, struct ether_addr *bbdev_ports_eth_address)
 282 {
 283         printf("Port %u, MAC address: %02X:%02X:%02X:%02X:%02X:%02X\n\n",
 284                         (unsigned int) portid,
 285                         bbdev_ports_eth_address->addr_bytes[0],
 286                         bbdev_ports_eth_address->addr_bytes[1],
 287                         bbdev_ports_eth_address->addr_bytes[2],
 288                         bbdev_ports_eth_address->addr_bytes[3],
 289                         bbdev_ports_eth_address->addr_bytes[4],
 290                         bbdev_ports_eth_address->addr_bytes[5]);
 291 }
 292
 293 static inline void
 294 pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int nb_to_free)
 295 {
 296         unsigned int i;
 297         for (i = 0; i < nb_to_free; ++i)
 298                 rte_pktmbuf_free(mbufs[i]);
 299 }
 300
 301 static inline void
 302 pktmbuf_userdata_free_bulk(struct rte_mbuf **mbufs, unsigned int nb_to_free)
 303 {
 304         unsigned int i;
 305         for (i = 0; i < nb_to_free; ++i) {
 306                 struct rte_mbuf *rx_pkt = mbufs[i]->userdata;
 307                 rte_pktmbuf_free(rx_pkt);
 308                 rte_pktmbuf_free(mbufs[i]);
 309         }
 310 }
 311
 312 /* Check the link status of all ports in up to 9s, and print them finally */
 313 static int
 314 check_port_link_status(uint16_t port_id)
 315 {
 316 #define CHECK_INTERVAL 100 /* 100ms */
 317 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
 318         uint8_t count;
 319         struct rte_eth_link link;
 320
 321         printf("\nChecking link status.");
 322         fflush(stdout);
 323
 324         for (count = 0; count <= MAX_CHECK_TIME &&
 325                         !rte_atomic16_read(&global_exit_flag); count++) {
 326                 memset(&link, 0, sizeof(link));
 327                 rte_eth_link_get_nowait(port_id, &link);
 328
 329                 if (link.link_status) {
 330                         const char *dp = (link.link_duplex ==
 331                                 ETH_LINK_FULL_DUPLEX) ?
 332                                 "full-duplex" : "half-duplex";
 333                         printf("\nPort %u Link Up - speed %u Mbps - %s\n",
 334                                 port_id, link.link_speed, dp);
 335                         return 0;
 336                 }
 337                 printf(".");
 338                 fflush(stdout);
 339                 rte_delay_ms(CHECK_INTERVAL);
 340         }
 341
 342         printf("\nPort %d Link Down\n", port_id);
 343         return 0;
 344 }
 345
 346 static inline void
 347 add_ether_hdr(struct rte_mbuf *pkt_src, struct rte_mbuf *pkt_dst)
 348 {
 349         struct ether_hdr *eth_from;
 350         struct ether_hdr *eth_to;
 351
 352         eth_from = rte_pktmbuf_mtod(pkt_src, struct ether_hdr *);
 353         eth_to = rte_pktmbuf_mtod(pkt_dst, struct ether_hdr *);
 354
 355         /* copy header */
 356         rte_memcpy(eth_to, eth_from, sizeof(struct ether_hdr));
 357 }
 358
 359 static inline void
 360 add_awgn(struct rte_mbuf **mbufs, uint16_t num_pkts)
 361 {
 362         RTE_SET_USED(mbufs);
 363         RTE_SET_USED(num_pkts);
 364 }
 365
 366 /* Encoder output to Decoder input adapter. The Decoder accepts only soft input
 367  * so each bit of the encoder output must be translated into one byte of LLR. If
 368  * Sub-block Deinterleaver is bypassed, which is the case, the padding bytes
 369  * must additionally be insterted at the end of each sub-block.
 370  */
 371 static inline void
 372 transform_enc_out_dec_in(struct rte_mbuf **mbufs, uint8_t *temp_buf,
 373                 uint16_t num_pkts, uint16_t k)
 374 {
 375         uint16_t i, l, j;
 376         uint16_t start_bit_idx;
 377         uint16_t out_idx;
 378         uint16_t d = k + 4;
 379         uint16_t kpi = RTE_ALIGN_CEIL(d, 32);
 380         uint16_t nd = kpi - d;
 381         uint16_t ncb = 3 * kpi;
 382
 383         for (i = 0; i < num_pkts; ++i) {
 384                 uint16_t pkt_data_len = rte_pktmbuf_data_len(mbufs[i]) -
 385                                 sizeof(struct ether_hdr);
 386
 387                 /* Resize the packet if needed */
 388                 if (pkt_data_len < ncb) {
 389                         char *data = rte_pktmbuf_append(mbufs[i],
 390                                         ncb - pkt_data_len);
 391                         if (data == NULL)
 392                                 printf(
 393                                         "Not enough space in decoder input packet");
 394                 }
 395
 396                 /* Translate each bit into 1 LLR byte. */
 397                 start_bit_idx = 0;
 398                 out_idx = 0;
 399                 for (j = 0; j < 3; ++j) {
 400                         for (l = start_bit_idx; l < start_bit_idx + d; ++l) {
 401                                 uint8_t *data = rte_pktmbuf_mtod_offset(
 402                                         mbufs[i], uint8_t *,
 403                                         sizeof(struct ether_hdr) + (l >> 3));
 404                                 if (*data & (0x80 >> (l & 7)))
 405                                         temp_buf[out_idx] = LLR_1_BIT;
 406                                 else
 407                                         temp_buf[out_idx] = LLR_0_BIT;
 408                                 ++out_idx;
 409                         }
 410                         /* Padding bytes should be at the end of the sub-block.
 411                          */
 412                         memset(&temp_buf[out_idx], 0, nd);
 413                         out_idx += nd;
 414                         start_bit_idx += d;
 415                 }
 416
 417                 rte_memcpy(rte_pktmbuf_mtod_offset(mbufs[i], uint8_t *,
 418                                 sizeof(struct ether_hdr)), temp_buf, ncb);
 419         }
 420 }
 421
 422 static inline void
 423 verify_data(struct rte_mbuf **mbufs, uint16_t num_pkts)
 424 {
 425         uint16_t i;
 426         for (i = 0; i < num_pkts; ++i) {
 427                 struct rte_mbuf *out = mbufs[i];
 428                 struct rte_mbuf *in = out->userdata;
 429
 430                 if (memcmp(rte_pktmbuf_mtod_offset(in, uint8_t *,
 431                                 sizeof(struct ether_hdr)),
 432                                 rte_pktmbuf_mtod_offset(out, uint8_t *,
 433                                 sizeof(struct ether_hdr)),
 434                                 K / 8 - CRC_24B_LEN))
 435                         printf("Input and output buffers are not equal!\n");
 436         }
 437 }
 438
 439 static int
 440 initialize_ports(struct app_config_params *app_params,
 441                 struct rte_mempool *ethdev_mbuf_mempool)
 442 {
 443         int ret;
 444         uint16_t port_id = app_params->port_id;
 445         uint16_t q;
 446         /* ethernet addresses of ports */
 447         struct ether_addr bbdev_port_eth_addr;
 448
 449         /* initialize ports */
 450         printf("\nInitializing port %u...\n", app_params->port_id);
 451         ret = rte_eth_dev_configure(port_id, app_params->num_enc_cores,
 452                 app_params->num_dec_cores, &port_conf);
 453
 454         if (ret < 0) {
 455                 printf("Cannot configure device: err=%d, port=%u\n",
 456                         ret, port_id);
 457                 return -1;
 458         }
 459
 460         /* initialize RX queues for encoder */
 461         for (q = 0; q < app_params->num_enc_cores; q++) {
 462                 ret = rte_eth_rx_queue_setup(port_id, q,
 463                         RTE_TEST_RX_DESC_DEFAULT,
 464                         rte_eth_dev_socket_id(port_id),
 465                         NULL, ethdev_mbuf_mempool);
 466                 if (ret < 0) {
 467                         printf("rte_eth_rx_queue_setup: err=%d, queue=%u\n",
 468                                 ret, q);
 469                         return -1;
 470                 }
 471         }
 472         /* initialize TX queues for decoder */
 473         for (q = 0; q < app_params->num_dec_cores; q++) {
 474                 ret = rte_eth_tx_queue_setup(port_id, q,
 475                         RTE_TEST_TX_DESC_DEFAULT,
 476                         rte_eth_dev_socket_id(port_id), NULL);
 477                 if (ret < 0) {
 478                         printf("rte_eth_tx_queue_setup: err=%d, queue=%u\n",
 479                                 ret, q);
 480                         return -1;
 481                 }
 482         }
 483
 484         rte_eth_promiscuous_enable(port_id);
 485
 486         rte_eth_macaddr_get(port_id, &bbdev_port_eth_addr);
 487         print_mac(port_id, &bbdev_port_eth_addr);
 488
 489         return 0;
 490 }
 491
 492 static void
 493 lcore_conf_init(struct app_config_params *app_params,
 494                 struct lcore_conf *lcore_conf,
 495                 struct rte_mempool **bbdev_op_pools,
 496                 struct rte_mempool *bbdev_mbuf_mempool,
 497                 struct rte_ring *enc_to_dec_ring,
 498                 struct lcore_statistics *lcore_stats)
 499 {
 500         unsigned int lcore_id;
 501         struct lcore_conf *lconf;
 502         uint16_t rx_queue_id = 0;
 503         uint16_t tx_queue_id = 0;
 504         uint16_t enc_q_id = 0;
 505         uint16_t dec_q_id = 0;
 506
 507         /* Configure lcores */
 508         for (lcore_id = 0; lcore_id < 8 * sizeof(uint64_t); ++lcore_id) {
 509                 lconf = &lcore_conf[lcore_id];
 510                 lconf->core_type = 0;
 511
 512                 if ((1ULL << lcore_id) & app_params->enc_core_mask) {
 513                         lconf->core_type |= (1 << RTE_BBDEV_OP_TURBO_ENC);
 514                         lconf->rx_queue_id = rx_queue_id++;
 515                         lconf->enc_queue_id =
 516                                         app_params->enc_queue_ids[enc_q_id++];
 517                 }
 518
 519                 if ((1ULL << lcore_id) & app_params->dec_core_mask) {
 520                         lconf->core_type |= (1 << RTE_BBDEV_OP_TURBO_DEC);
 521                         lconf->tx_queue_id = tx_queue_id++;
 522                         lconf->dec_queue_id =
 523                                         app_params->dec_queue_ids[dec_q_id++];
 524                 }
 525
 526                 lconf->bbdev_enc_op_pool =
 527                                 bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC];
 528                 lconf->bbdev_dec_op_pool =
 529                                 bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC];
 530                 lconf->bbdev_id = app_params->bbdev_id;
 531                 lconf->port_id = app_params->port_id;
 532                 lconf->enc_out_pool = bbdev_mbuf_mempool;
 533                 lconf->enc_to_dec_ring = enc_to_dec_ring;
 534                 lconf->lcore_stats = &lcore_stats[lcore_id];
 535         }
 536 }
 537
 538 static void
 539 print_lcore_stats(struct lcore_statistics *lstats, unsigned int lcore_id)
 540 {
 541         static const char *stats_border = "_______";
 542
 543         printf("\nLcore %d: %s enqueued count:\t\t%u\n",
 544                         lcore_id, stats_border, lstats->enqueued);
 545         printf("Lcore %d: %s dequeued count:\t\t%u\n",
 546                         lcore_id, stats_border, lstats->dequeued);
 547         printf("Lcore %d: %s RX lost packets count:\t\t%u\n",
 548                         lcore_id, stats_border, lstats->rx_lost_packets);
 549         printf("Lcore %d: %s encoder-to-decoder lost count:\t%u\n",
 550                         lcore_id, stats_border,
 551                         lstats->enc_to_dec_lost_packets);
 552         printf("Lcore %d: %s TX lost packets count:\t\t%u\n",
 553                         lcore_id, stats_border, lstats->tx_lost_packets);
 554 }
 555
 556 static void
 557 print_stats(struct stats_lcore_params *stats_lcore)
 558 {
 559         unsigned int l_id;
 560         unsigned int bbdev_id = stats_lcore->app_params->bbdev_id;
 561         unsigned int port_id = stats_lcore->app_params->port_id;
 562         int len, ret, i;
 563
 564         struct rte_eth_xstat *xstats;
 565         struct rte_eth_xstat_name *xstats_names;
 566         struct rte_bbdev_stats bbstats;
 567         static const char *stats_border = "_______";
 568
 569         const char clr[] = { 27, '[', '2', 'J', '\0' };
 570         const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0' };
 571
 572         /* Clear screen and move to top left */
 573         printf("%s%s", clr, topLeft);
 574
 575         printf("PORT STATISTICS:\n================\n");
 576         len = rte_eth_xstats_get(port_id, NULL, 0);
 577         if (len < 0)
 578                 rte_exit(EXIT_FAILURE,
 579                                 "rte_eth_xstats_get(%u) failed: %d", port_id,
 580                                 len);
 581
 582         xstats = calloc(len, sizeof(*xstats));
 583         if (xstats == NULL)
 584                 rte_exit(EXIT_FAILURE,
 585                                 "Failed to calloc memory for xstats");
 586
 587         ret = rte_eth_xstats_get(port_id, xstats, len);
 588         if (ret < 0 || ret > len) {
 589                 free(xstats);
 590                 rte_exit(EXIT_FAILURE,
 591                                 "rte_eth_xstats_get(%u) len%i failed: %d",
 592                                 port_id, len, ret);
 593         }
 594
 595         xstats_names = calloc(len, sizeof(*xstats_names));
 596         if (xstats_names == NULL) {
 597                 free(xstats);
 598                 rte_exit(EXIT_FAILURE,
 599                                 "Failed to calloc memory for xstats_names");
 600         }
 601
 602         ret = rte_eth_xstats_get_names(port_id, xstats_names, len);
 603         if (ret < 0 || ret > len) {
 604                 free(xstats);
 605                 free(xstats_names);
 606                 rte_exit(EXIT_FAILURE,
 607                                 "rte_eth_xstats_get_names(%u) len%i failed: %d",
 608                                 port_id, len, ret);
 609         }
 610
 611         for (i = 0; i < len; i++) {
 612                 if (xstats[i].value > 0)
 613                         printf("Port %u: %s %s:\t\t%"PRIu64"\n",
 614                                         port_id, stats_border,
 615                                         xstats_names[i].name,
 616                                         xstats[i].value);
 617         }
 618
 619         ret = rte_bbdev_stats_get(bbdev_id, &bbstats);
 620         if (ret < 0) {
 621                 free(xstats);
 622                 free(xstats_names);
 623                 rte_exit(EXIT_FAILURE,
 624                                 "ERROR(%d): Failure to get BBDEV %u statistics\n",
 625                                 ret, bbdev_id);
 626         }
 627
 628         printf("\nBBDEV STATISTICS:\n=================\n");
 629         printf("BBDEV %u: %s enqueue count:\t\t%"PRIu64"\n",
 630                         bbdev_id, stats_border,
 631                         bbstats.enqueued_count);
 632         printf("BBDEV %u: %s dequeue count:\t\t%"PRIu64"\n",
 633                         bbdev_id, stats_border,
 634                         bbstats.dequeued_count);
 635         printf("BBDEV %u: %s enqueue error count:\t\t%"PRIu64"\n",
 636                         bbdev_id, stats_border,
 637                         bbstats.enqueue_err_count);
 638         printf("BBDEV %u: %s dequeue error count:\t\t%"PRIu64"\n\n",
 639                         bbdev_id, stats_border,
 640                         bbstats.dequeue_err_count);
 641
 642         printf("LCORE STATISTICS:\n=================\n");
 643         for (l_id = 0; l_id < RTE_MAX_LCORE; ++l_id) {
 644                 if (stats_lcore->lconf[l_id].core_type == 0)
 645                         continue;
 646                 print_lcore_stats(stats_lcore->lconf[l_id].lcore_stats, l_id);
 647         }
 648
 649         free(xstats);
 650         free(xstats_names);
 651 }
 652
 653 static int
 654 stats_loop(void *arg)
 655 {
 656         struct stats_lcore_params *stats_lcore = arg;
 657
 658         while (!rte_atomic16_read(&global_exit_flag)) {
 659                 print_stats(stats_lcore);
 660                 rte_delay_ms(500);
 661         }
 662
 663         return 0;
 664 }
 665
 666 static inline void
 667 run_encoding(struct lcore_conf *lcore_conf)
 668 {
 669         uint16_t i;
 670         uint16_t port_id, rx_queue_id;
 671         uint16_t bbdev_id, enc_queue_id;
 672         uint16_t nb_rx, nb_enq, nb_deq, nb_sent;
 673         struct rte_mbuf *rx_pkts_burst[MAX_PKT_BURST];
 674         struct rte_mbuf *enc_out_pkts[MAX_PKT_BURST];
 675         struct rte_bbdev_enc_op *bbdev_ops_burst[MAX_PKT_BURST];
 676         struct lcore_statistics *lcore_stats;
 677         struct rte_mempool *bbdev_op_pool, *enc_out_pool;
 678         struct rte_ring *enc_to_dec_ring;
 679         const int in_data_len = (def_op_enc.cb_params.k / 8) - CRC_24B_LEN;
 680
 681         lcore_stats = lcore_conf->lcore_stats;
 682         port_id = lcore_conf->port_id;
 683         rx_queue_id = lcore_conf->rx_queue_id;
 684         bbdev_id = lcore_conf->bbdev_id;
 685         enc_queue_id = lcore_conf->enc_queue_id;
 686         bbdev_op_pool = lcore_conf->bbdev_enc_op_pool;
 687         enc_out_pool = lcore_conf->enc_out_pool;
 688         enc_to_dec_ring = lcore_conf->enc_to_dec_ring;
 689
 690         /* Read packet from RX queues*/
 691         nb_rx = rte_eth_rx_burst(port_id, rx_queue_id, rx_pkts_burst,
 692                         MAX_PKT_BURST);
 693         if (!nb_rx)
 694                 return;
 695
 696         if (unlikely(rte_mempool_get_bulk(enc_out_pool, (void **)enc_out_pkts,
 697                         nb_rx) != 0)) {
 698                 pktmbuf_free_bulk(rx_pkts_burst, nb_rx);
 699                 lcore_stats->rx_lost_packets += nb_rx;
 700                 return;
 701         }
 702
 703         if (unlikely(rte_bbdev_enc_op_alloc_bulk(bbdev_op_pool, bbdev_ops_burst,
 704                         nb_rx) != 0)) {
 705                 pktmbuf_free_bulk(enc_out_pkts, nb_rx);
 706                 pktmbuf_free_bulk(rx_pkts_burst, nb_rx);
 707                 lcore_stats->rx_lost_packets += nb_rx;
 708                 return;
 709         }
 710
 711         for (i = 0; i < nb_rx; i++) {
 712                 char *data;
 713                 const uint16_t pkt_data_len =
 714                                 rte_pktmbuf_data_len(rx_pkts_burst[i]) -
 715                                 sizeof(struct ether_hdr);
 716                 /* save input mbuf pointer for later comparison */
 717                 enc_out_pkts[i]->userdata = rx_pkts_burst[i];
 718
 719                 /* copy ethernet header */
 720                 rte_pktmbuf_reset(enc_out_pkts[i]);
 721                 data = rte_pktmbuf_append(enc_out_pkts[i],
 722                                 sizeof(struct ether_hdr));
 723                 if (data == NULL) {
 724                         printf(
 725                                 "Not enough space for ethernet header in encoder output mbuf\n");
 726                         continue;
 727                 }
 728                 add_ether_hdr(rx_pkts_burst[i], enc_out_pkts[i]);
 729
 730                 /* set op */
 731                 bbdev_ops_burst[i]->turbo_enc = def_op_enc;
 732
 733                 bbdev_ops_burst[i]->turbo_enc.input.data =
 734                                 rx_pkts_burst[i];
 735                 bbdev_ops_burst[i]->turbo_enc.input.offset =
 736                                 sizeof(struct ether_hdr);
 737                 /* Encoder will attach the CRC24B, adjust the length */
 738                 bbdev_ops_burst[i]->turbo_enc.input.length = in_data_len;
 739
 740                 if (in_data_len < pkt_data_len)
 741                         rte_pktmbuf_trim(rx_pkts_burst[i], pkt_data_len -
 742                                         in_data_len);
 743                 else if (in_data_len > pkt_data_len) {
 744                         data = rte_pktmbuf_append(rx_pkts_burst[i],
 745                                         in_data_len - pkt_data_len);
 746                         if (data == NULL)
 747                                 printf(
 748                                         "Not enough storage in mbuf to perform the encoding\n");
 749                 }
 750
 751                 bbdev_ops_burst[i]->turbo_enc.output.data =
 752                                 enc_out_pkts[i];
 753                 bbdev_ops_burst[i]->turbo_enc.output.offset =
 754                                 sizeof(struct ether_hdr);
 755         }
 756
 757         /* Enqueue packets on BBDevice */
 758         nb_enq = rte_bbdev_enqueue_enc_ops(bbdev_id, enc_queue_id,
 759                         bbdev_ops_burst, nb_rx);
 760         if (unlikely(nb_enq < nb_rx)) {
 761                 pktmbuf_userdata_free_bulk(&enc_out_pkts[nb_enq],
 762                                 nb_rx - nb_enq);
 763                 rte_bbdev_enc_op_free_bulk(&bbdev_ops_burst[nb_enq],
 764                                 nb_rx - nb_enq);
 765                 lcore_stats->rx_lost_packets += nb_rx - nb_enq;
 766
 767                 if (!nb_enq)
 768                         return;
 769         }
 770
 771         lcore_stats->enqueued += nb_enq;
 772
 773         /* Dequeue packets from bbdev device*/
 774         nb_deq = 0;
 775         do {
 776                 nb_deq += rte_bbdev_dequeue_enc_ops(bbdev_id, enc_queue_id,
 777                                 &bbdev_ops_burst[nb_deq], nb_enq - nb_deq);
 778         } while (unlikely(nb_deq < nb_enq));
 779
 780         lcore_stats->dequeued += nb_deq;
 781
 782         /* Generate and add AWGN */
 783         add_awgn(enc_out_pkts, nb_deq);
 784
 785         rte_bbdev_enc_op_free_bulk(bbdev_ops_burst, nb_deq);
 786
 787         /* Enqueue packets to encoder-to-decoder ring */
 788         nb_sent = rte_ring_enqueue_burst(enc_to_dec_ring, (void **)enc_out_pkts,
 789                         nb_deq, NULL);
 790         if (unlikely(nb_sent < nb_deq)) {
 791                 pktmbuf_userdata_free_bulk(&enc_out_pkts[nb_sent],
 792                                 nb_deq - nb_sent);
 793                 lcore_stats->enc_to_dec_lost_packets += nb_deq - nb_sent;
 794         }
 795 }
 796
 797 static void
 798 run_decoding(struct lcore_conf *lcore_conf)
 799 {
 800         uint16_t i;
 801         uint16_t port_id, tx_queue_id;
 802         uint16_t bbdev_id, bbdev_queue_id;
 803         uint16_t nb_recv, nb_enq, nb_deq, nb_tx;
 804         uint8_t *llr_temp_buf;
 805         struct rte_mbuf *recv_pkts_burst[MAX_PKT_BURST];
 806         struct rte_bbdev_dec_op *bbdev_ops_burst[MAX_PKT_BURST];
 807         struct lcore_statistics *lcore_stats;
 808         struct rte_mempool *bbdev_op_pool;
 809         struct rte_ring *enc_to_dec_ring;
 810
 811         lcore_stats = lcore_conf->lcore_stats;
 812         port_id = lcore_conf->port_id;
 813         tx_queue_id = lcore_conf->tx_queue_id;
 814         bbdev_id = lcore_conf->bbdev_id;
 815         bbdev_queue_id = lcore_conf->dec_queue_id;
 816         bbdev_op_pool = lcore_conf->bbdev_dec_op_pool;
 817         enc_to_dec_ring = lcore_conf->enc_to_dec_ring;
 818         llr_temp_buf = lcore_conf->llr_temp_buf;
 819
 820         /* Dequeue packets from the ring */
 821         nb_recv = rte_ring_dequeue_burst(enc_to_dec_ring,
 822                         (void **)recv_pkts_burst, MAX_PKT_BURST, NULL);
 823         if (!nb_recv)
 824                 return;
 825
 826         if (unlikely(rte_bbdev_dec_op_alloc_bulk(bbdev_op_pool, bbdev_ops_burst,
 827                         nb_recv) != 0)) {
 828                 pktmbuf_userdata_free_bulk(recv_pkts_burst, nb_recv);
 829                 lcore_stats->rx_lost_packets += nb_recv;
 830                 return;
 831         }
 832
 833         transform_enc_out_dec_in(recv_pkts_burst, llr_temp_buf, nb_recv,
 834                         def_op_dec.cb_params.k);
 835
 836         for (i = 0; i < nb_recv; i++) {
 837                 /* set op */
 838                 bbdev_ops_burst[i]->turbo_dec = def_op_dec;
 839
 840                 bbdev_ops_burst[i]->turbo_dec.input.data = recv_pkts_burst[i];
 841                 bbdev_ops_burst[i]->turbo_dec.input.offset =
 842                                 sizeof(struct ether_hdr);
 843                 bbdev_ops_burst[i]->turbo_dec.input.length =
 844                                 rte_pktmbuf_data_len(recv_pkts_burst[i])
 845                                 - sizeof(struct ether_hdr);
 846
 847                 bbdev_ops_burst[i]->turbo_dec.hard_output.data =
 848                                 recv_pkts_burst[i];
 849                 bbdev_ops_burst[i]->turbo_dec.hard_output.offset =
 850                                 sizeof(struct ether_hdr);
 851         }
 852
 853         /* Enqueue packets on BBDevice */
 854         nb_enq = rte_bbdev_enqueue_dec_ops(bbdev_id, bbdev_queue_id,
 855                         bbdev_ops_burst, nb_recv);
 856         if (unlikely(nb_enq < nb_recv)) {
 857                 pktmbuf_userdata_free_bulk(&recv_pkts_burst[nb_enq],
 858                                 nb_recv - nb_enq);
 859                 rte_bbdev_dec_op_free_bulk(&bbdev_ops_burst[nb_enq],
 860                                 nb_recv - nb_enq);
 861                 lcore_stats->rx_lost_packets += nb_recv - nb_enq;
 862
 863                 if (!nb_enq)
 864                         return;
 865         }
 866
 867         lcore_stats->enqueued += nb_enq;
 868
 869         /* Dequeue packets from BBDevice */
 870         nb_deq = 0;
 871         do {
 872                 nb_deq += rte_bbdev_dequeue_dec_ops(bbdev_id, bbdev_queue_id,
 873                                 &bbdev_ops_burst[nb_deq], nb_enq - nb_deq);
 874         } while (unlikely(nb_deq < nb_enq));
 875
 876         lcore_stats->dequeued += nb_deq;
 877
 878         rte_bbdev_dec_op_free_bulk(bbdev_ops_burst, nb_deq);
 879
 880         verify_data(recv_pkts_burst, nb_deq);
 881
 882         /* Free the RX mbufs after verification */
 883         for (i = 0; i < nb_deq; ++i)
 884                 rte_pktmbuf_free(recv_pkts_burst[i]->userdata);
 885
 886         /* Transmit the packets */
 887         nb_tx = rte_eth_tx_burst(port_id, tx_queue_id, recv_pkts_burst, nb_deq);
 888         if (unlikely(nb_tx < nb_deq)) {
 889                 pktmbuf_userdata_free_bulk(&recv_pkts_burst[nb_tx],
 890                                 nb_deq - nb_tx);
 891                 lcore_stats->tx_lost_packets += nb_deq - nb_tx;
 892         }
 893 }
 894
 895 static int
 896 processing_loop(void *arg)
 897 {
 898         struct lcore_conf *lcore_conf = arg;
 899         const bool run_encoder = (lcore_conf->core_type &
 900                         (1 << RTE_BBDEV_OP_TURBO_ENC));
 901         const bool run_decoder = (lcore_conf->core_type &
 902                         (1 << RTE_BBDEV_OP_TURBO_DEC));
 903
 904         while (!rte_atomic16_read(&global_exit_flag)) {
 905                 if (run_encoder)
 906                         run_encoding(lcore_conf);
 907                 if (run_decoder)
 908                         run_decoding(lcore_conf);
 909         }
 910
 911         return 0;
 912 }
 913
 914 static int
 915 prepare_bbdev_device(unsigned int dev_id, struct rte_bbdev_info *info,
 916                 struct app_config_params *app_params)
 917 {
 918         int ret;
 919         unsigned int q_id, dec_q_id, enc_q_id;
 920         struct rte_bbdev_queue_conf qconf = {0};
 921         uint16_t dec_qs_nb = app_params->num_dec_cores;
 922         uint16_t enc_qs_nb = app_params->num_enc_cores;
 923         uint16_t tot_qs = dec_qs_nb + enc_qs_nb;
 924
 925         ret = rte_bbdev_setup_queues(dev_id, tot_qs, info->socket_id);
 926         if (ret < 0)
 927                 rte_exit(EXIT_FAILURE,
 928                                 "ERROR(%d): BBDEV %u not configured properly\n",
 929                                 ret, dev_id);
 930
 931         /* setup device DEC queues */
 932         qconf.socket = info->socket_id;
 933         qconf.queue_size = info->drv.queue_size_lim;
 934         qconf.op_type = RTE_BBDEV_OP_TURBO_DEC;
 935
 936         for (q_id = 0, dec_q_id = 0; q_id < dec_qs_nb; q_id++) {
 937                 ret = rte_bbdev_queue_configure(dev_id, q_id, &qconf);
 938                 if (ret < 0)
 939                         rte_exit(EXIT_FAILURE,
 940                                         "ERROR(%d): BBDEV %u DEC queue %u not configured properly\n",
 941                                         ret, dev_id, q_id);
 942                 app_params->dec_queue_ids[dec_q_id++] = q_id;
 943         }
 944
 945         /* setup device ENC queues */
 946         qconf.op_type = RTE_BBDEV_OP_TURBO_ENC;
 947
 948         for (q_id = dec_qs_nb, enc_q_id = 0; q_id < tot_qs; q_id++) {
 949                 ret = rte_bbdev_queue_configure(dev_id, q_id, &qconf);
 950                 if (ret < 0)
 951                         rte_exit(EXIT_FAILURE,
 952                                         "ERROR(%d): BBDEV %u ENC queue %u not configured properly\n",
 953                                         ret, dev_id, q_id);
 954                 app_params->enc_queue_ids[enc_q_id++] = q_id;
 955         }
 956
 957         ret = rte_bbdev_start(dev_id);
 958
 959         if (ret != 0)
 960                 rte_exit(EXIT_FAILURE, "ERROR(%d): BBDEV %u not started\n",
 961                         ret, dev_id);
 962
 963         printf("BBdev %u started\n", dev_id);
 964
 965         return 0;
 966 }
 967
 968 static inline bool
 969 check_matching_capabilities(uint64_t mask, uint64_t required_mask)
 970 {
 971         return (mask & required_mask) == required_mask;
 972 }
 973
 974 static void
 975 enable_bbdev(struct app_config_params *app_params)
 976 {
 977         struct rte_bbdev_info dev_info;
 978         const struct rte_bbdev_op_cap *op_cap;
 979         uint16_t bbdev_id = app_params->bbdev_id;
 980         bool encoder_capable = false;
 981         bool decoder_capable = false;
 982
 983         rte_bbdev_info_get(bbdev_id, &dev_info);
 984         op_cap = dev_info.drv.capabilities;
 985
 986         while (op_cap->type != RTE_BBDEV_OP_NONE) {
 987                 if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) {
 988                         if (check_matching_capabilities(
 989                                         op_cap->cap.turbo_enc.capability_flags,
 990                                         def_op_enc.op_flags))
 991                                 encoder_capable = true;
 992                 }
 993
 994                 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) {
 995                         if (check_matching_capabilities(
 996                                         op_cap->cap.turbo_dec.capability_flags,
 997                                         def_op_dec.op_flags))
 998                                 decoder_capable = true;
 999                 }
1000
1001                 op_cap++;
1002         }
1003
1004         if (encoder_capable == false)
1005                 rte_exit(EXIT_FAILURE,
1006                         "The specified BBDev %u doesn't have required encoder capabilities!\n",
1007                         bbdev_id);
1008         if (decoder_capable == false)
1009                 rte_exit(EXIT_FAILURE,
1010                         "The specified BBDev %u doesn't have required decoder capabilities!\n",
1011                         bbdev_id);
1012
1013         prepare_bbdev_device(bbdev_id, &dev_info, app_params);
1014 }
1015
1016 int
1017 main(int argc, char **argv)
1018 {
1019         int ret;
1020         unsigned int nb_bbdevs, nb_ports, flags, lcore_id;
1021         void *sigret;
1022         struct app_config_params app_params = def_app_config;
1023         struct rte_mempool *ethdev_mbuf_mempool, *bbdev_mbuf_mempool;
1024         struct rte_mempool *bbdev_op_pools[RTE_BBDEV_OP_TYPE_COUNT];
1025         struct lcore_conf lcore_conf[RTE_MAX_LCORE] = { {0} };
1026         struct lcore_statistics lcore_stats[RTE_MAX_LCORE] = { {0} };
1027         struct stats_lcore_params stats_lcore;
1028         struct rte_ring *enc_to_dec_ring;
1029         bool stats_thread_started = false;
1030         unsigned int master_lcore_id = rte_get_master_lcore();
1031
1032         rte_atomic16_init(&global_exit_flag);
1033
1034         sigret = signal(SIGTERM, signal_handler);
1035         if (sigret == SIG_ERR)
1036                 rte_exit(EXIT_FAILURE, "signal(%d, ...) failed", SIGTERM);
1037
1038         sigret = signal(SIGINT, signal_handler);
1039         if (sigret == SIG_ERR)
1040                 rte_exit(EXIT_FAILURE, "signal(%d, ...) failed", SIGINT);
1041
1042         ret = rte_eal_init(argc, argv);
1043         if (ret < 0)
1044                 rte_exit(EXIT_FAILURE, "Invalid EAL arguments\n");
1045
1046         argc -= ret;
1047         argv += ret;
1048
1049         /* parse application arguments (after the EAL ones) */
1050         ret = bbdev_parse_args(argc, argv, &app_params);
1051         if (ret < 0)
1052                 rte_exit(EXIT_FAILURE, "Invalid BBDEV arguments\n");
1053
1054         /*create bbdev op pools*/
1055         bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC] =
1056                         rte_bbdev_op_pool_create("bbdev_op_pool_dec",
1057                         RTE_BBDEV_OP_TURBO_DEC, NB_MBUF, 128, rte_socket_id());
1058         bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC] =
1059                         rte_bbdev_op_pool_create("bbdev_op_pool_enc",
1060                         RTE_BBDEV_OP_TURBO_ENC, NB_MBUF, 128, rte_socket_id());
1061
1062         if ((bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC] == NULL) ||
1063                         (bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC] == NULL))
1064                 rte_exit(EXIT_FAILURE, "Cannot create bbdev op pools\n");
1065
1066         /* Create encoder to decoder ring */
1067         flags = (app_params.num_enc_cores == 1) ? RING_F_SP_ENQ : 0;
1068         if (app_params.num_dec_cores == 1)
1069                 flags |= RING_F_SC_DEQ;
1070
1071         enc_to_dec_ring = rte_ring_create("enc_to_dec_ring",
1072                 rte_align32pow2(NB_MBUF), rte_socket_id(), flags);
1073
1074         /* Get the number of available bbdev devices */
1075         nb_bbdevs = rte_bbdev_count();
1076         if (nb_bbdevs <= app_params.bbdev_id)
1077                 rte_exit(EXIT_FAILURE,
1078                                 "%u BBDevs detected, cannot use BBDev with ID %u!\n",
1079                                 nb_bbdevs, app_params.bbdev_id);
1080         printf("Number of bbdevs detected: %d\n", nb_bbdevs);
1081
1082         /* Get the number of available ethdev devices */
1083         nb_ports = rte_eth_dev_count();
1084         if (nb_ports <= app_params.port_id)
1085                 rte_exit(EXIT_FAILURE,
1086                                 "%u ports detected, cannot use port with ID %u!\n",
1087                                 nb_ports, app_params.port_id);
1088
1089         /* create the mbuf mempool for ethdev pkts */
1090         ethdev_mbuf_mempool = rte_pktmbuf_pool_create("ethdev_mbuf_pool",
1091                         NB_MBUF, MEMPOOL_CACHE_SIZE, 0,
1092                         RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
1093         if (ethdev_mbuf_mempool == NULL)
1094                 rte_exit(EXIT_FAILURE, "Cannot create ethdev mbuf mempool\n");
1095
1096         /* create the mbuf mempool for encoder output */
1097         bbdev_mbuf_mempool = rte_pktmbuf_pool_create("bbdev_mbuf_pool",
1098                         NB_MBUF, MEMPOOL_CACHE_SIZE, 0,
1099                         RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
1100         if (bbdev_mbuf_mempool == NULL)
1101                 rte_exit(EXIT_FAILURE, "Cannot create ethdev mbuf mempool\n");
1102
1103         /* initialize ports */
1104         ret = initialize_ports(&app_params, ethdev_mbuf_mempool);
1105
1106         /* Check if all requested lcores are available */
1107         for (lcore_id = 0; lcore_id < 8 * sizeof(uint64_t); ++lcore_id)
1108                 if (((1ULL << lcore_id) & app_params.enc_core_mask) ||
1109                                 ((1ULL << lcore_id) & app_params.dec_core_mask))
1110                         if (!rte_lcore_is_enabled(lcore_id))
1111                                 rte_exit(EXIT_FAILURE,
1112                                                 "Requested lcore_id %u is not enabled!\n",
1113                                                 lcore_id);
1114
1115         /* Start ethernet port */
1116         ret = rte_eth_dev_start(app_params.port_id);
1117         if (ret < 0)
1118                 rte_exit(EXIT_FAILURE, "rte_eth_dev_start:err=%d, port=%u\n",
1119                                 ret, app_params.port_id);
1120
1121         ret = check_port_link_status(app_params.port_id);
1122         if (ret < 0)
1123                 exit(EXIT_FAILURE);
1124
1125         /* start BBDevice and save BBDev queue IDs */
1126         enable_bbdev(&app_params);
1127
1128         /* Initialize the port/queue configuration of each logical core */
1129         lcore_conf_init(&app_params, lcore_conf, bbdev_op_pools,
1130                         bbdev_mbuf_mempool, enc_to_dec_ring, lcore_stats);
1131
1132         stats_lcore.app_params = &app_params;
1133         stats_lcore.lconf = lcore_conf;
1134
1135         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
1136                 if (lcore_conf[lcore_id].core_type != 0)
1137                         /* launch per-lcore processing loop on slave lcores */
1138                         rte_eal_remote_launch(processing_loop,
1139                                         &lcore_conf[lcore_id], lcore_id);
1140                 else if (!stats_thread_started) {
1141                         /* launch statistics printing loop */
1142                         rte_eal_remote_launch(stats_loop, &stats_lcore,
1143                                         lcore_id);
1144                         stats_thread_started = true;
1145                 }
1146         }
1147
1148         if (!stats_thread_started &&
1149                         lcore_conf[master_lcore_id].core_type != 0)
1150                 rte_exit(EXIT_FAILURE,
1151                                 "Not enough lcores to run the statistics printing loop!");
1152         else if (lcore_conf[master_lcore_id].core_type != 0)
1153                 processing_loop(&lcore_conf[master_lcore_id]);
1154         else if (!stats_thread_started)
1155                 stats_loop(&stats_lcore);
1156
1157         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
1158                 ret |= rte_eal_wait_lcore(lcore_id);
1159         }
1160
1161         return ret;
1162 }