examples/load_balancer/init.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2014 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <stdlib.h>
   7 #include <stdint.h>
   8 #include <inttypes.h>
   9 #include <sys/types.h>
  10 #include <string.h>
  11 #include <sys/queue.h>
  12 #include <stdarg.h>
  13 #include <errno.h>
  14 #include <getopt.h>
  15
  16 #include <rte_common.h>
  17 #include <rte_byteorder.h>
  18 #include <rte_log.h>
  19 #include <rte_memory.h>
  20 #include <rte_memcpy.h>
  21 #include <rte_eal.h>
  22 #include <rte_launch.h>
  23 #include <rte_atomic.h>
  24 #include <rte_cycles.h>
  25 #include <rte_prefetch.h>
  26 #include <rte_lcore.h>
  27 #include <rte_per_lcore.h>
  28 #include <rte_branch_prediction.h>
  29 #include <rte_interrupts.h>
  30 #include <rte_random.h>
  31 #include <rte_debug.h>
  32 #include <rte_ether.h>
  33 #include <rte_ethdev.h>
  34 #include <rte_ring.h>
  35 #include <rte_mempool.h>
  36 #include <rte_mbuf.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_ip.h>
  39 #include <rte_tcp.h>
  40 #include <rte_lpm.h>
  41
  42 #include "main.h"
  43
  44 static struct rte_eth_conf port_conf = {
  45         .rxmode = {
  46                 .mq_mode        = ETH_MQ_RX_RSS,
  47                 .split_hdr_size = 0,
  48                 .offloads = (DEV_RX_OFFLOAD_CHECKSUM |
  49                              DEV_RX_OFFLOAD_CRC_STRIP),
  50         },
  51         .rx_adv_conf = {
  52                 .rss_conf = {
  53                         .rss_key = NULL,
  54                         .rss_hf = ETH_RSS_IP,
  55                 },
  56         },
  57         .txmode = {
  58                 .mq_mode = ETH_MQ_TX_NONE,
  59         },
  60 };
  61
  62 static void
  63 app_assign_worker_ids(void)
  64 {
  65         uint32_t lcore, worker_id;
  66
  67         /* Assign ID for each worker */
  68         worker_id = 0;
  69         for (lcore = 0; lcore < APP_MAX_LCORES; lcore ++) {
  70                 struct app_lcore_params_worker *lp_worker = &app.lcore_params[lcore].worker;
  71
  72                 if (app.lcore_params[lcore].type != e_APP_LCORE_WORKER) {
  73                         continue;
  74                 }
  75
  76                 lp_worker->worker_id = worker_id;
  77                 worker_id ++;
  78         }
  79 }
  80
  81 static void
  82 app_init_mbuf_pools(void)
  83 {
  84         unsigned socket, lcore;
  85
  86         /* Init the buffer pools */
  87         for (socket = 0; socket < APP_MAX_SOCKETS; socket ++) {
  88                 char name[32];
  89                 if (app_is_socket_used(socket) == 0) {
  90                         continue;
  91                 }
  92
  93                 snprintf(name, sizeof(name), "mbuf_pool_%u", socket);
  94                 printf("Creating the mbuf pool for socket %u ...\n", socket);
  95                 app.pools[socket] = rte_pktmbuf_pool_create(
  96                         name, APP_DEFAULT_MEMPOOL_BUFFERS,
  97                         APP_DEFAULT_MEMPOOL_CACHE_SIZE,
  98                         0, APP_DEFAULT_MBUF_DATA_SIZE, socket);
  99                 if (app.pools[socket] == NULL) {
 100                         rte_panic("Cannot create mbuf pool on socket %u\n", socket);
 101                 }
 102         }
 103
 104         for (lcore = 0; lcore < APP_MAX_LCORES; lcore ++) {
 105                 if (app.lcore_params[lcore].type == e_APP_LCORE_DISABLED) {
 106                         continue;
 107                 }
 108
 109                 socket = rte_lcore_to_socket_id(lcore);
 110                 app.lcore_params[lcore].pool = app.pools[socket];
 111         }
 112 }
 113
 114 static void
 115 app_init_lpm_tables(void)
 116 {
 117         unsigned socket, lcore;
 118
 119         /* Init the LPM tables */
 120         for (socket = 0; socket < APP_MAX_SOCKETS; socket ++) {
 121                 char name[32];
 122                 uint32_t rule;
 123
 124                 if (app_is_socket_used(socket) == 0) {
 125                         continue;
 126                 }
 127
 128                 struct rte_lpm_config lpm_config;
 129
 130                 lpm_config.max_rules = APP_MAX_LPM_RULES;
 131                 lpm_config.number_tbl8s = 256;
 132                 lpm_config.flags = 0;
 133                 snprintf(name, sizeof(name), "lpm_table_%u", socket);
 134                 printf("Creating the LPM table for socket %u ...\n", socket);
 135                 app.lpm_tables[socket] = rte_lpm_create(
 136                         name,
 137                         socket,
 138                         &lpm_config);
 139                 if (app.lpm_tables[socket] == NULL) {
 140                         rte_panic("Unable to create LPM table on socket %u\n", socket);
 141                 }
 142
 143                 for (rule = 0; rule < app.n_lpm_rules; rule ++) {
 144                         int ret;
 145
 146                         ret = rte_lpm_add(app.lpm_tables[socket],
 147                                 app.lpm_rules[rule].ip,
 148                                 app.lpm_rules[rule].depth,
 149                                 app.lpm_rules[rule].if_out);
 150
 151                         if (ret < 0) {
 152                                 rte_panic("Unable to add entry %u (%x/%u => %u) to the LPM table on socket %u (%d)\n",
 153                                         (unsigned) rule,
 154                                         (unsigned) app.lpm_rules[rule].ip,
 155                                         (unsigned) app.lpm_rules[rule].depth,
 156                                         (unsigned) app.lpm_rules[rule].if_out,
 157                                         socket,
 158                                         ret);
 159                         }
 160                 }
 161
 162         }
 163
 164         for (lcore = 0; lcore < APP_MAX_LCORES; lcore ++) {
 165                 if (app.lcore_params[lcore].type != e_APP_LCORE_WORKER) {
 166                         continue;
 167                 }
 168
 169                 socket = rte_lcore_to_socket_id(lcore);
 170                 app.lcore_params[lcore].worker.lpm_table = app.lpm_tables[socket];
 171         }
 172 }
 173
 174 static void
 175 app_init_rings_rx(void)
 176 {
 177         unsigned lcore;
 178
 179         /* Initialize the rings for the RX side */
 180         for (lcore = 0; lcore < APP_MAX_LCORES; lcore ++) {
 181                 struct app_lcore_params_io *lp_io = &app.lcore_params[lcore].io;
 182                 unsigned socket_io, lcore_worker;
 183
 184                 if ((app.lcore_params[lcore].type != e_APP_LCORE_IO) ||
 185                     (lp_io->rx.n_nic_queues == 0)) {
 186                         continue;
 187                 }
 188
 189                 socket_io = rte_lcore_to_socket_id(lcore);
 190
 191                 for (lcore_worker = 0; lcore_worker < APP_MAX_LCORES; lcore_worker ++) {
 192                         char name[32];
 193                         struct app_lcore_params_worker *lp_worker = &app.lcore_params[lcore_worker].worker;
 194                         struct rte_ring *ring = NULL;
 195
 196                         if (app.lcore_params[lcore_worker].type != e_APP_LCORE_WORKER) {
 197                                 continue;
 198                         }
 199
 200                         printf("Creating ring to connect I/O lcore %u (socket %u) with worker lcore %u ...\n",
 201                                 lcore,
 202                                 socket_io,
 203                                 lcore_worker);
 204                         snprintf(name, sizeof(name), "app_ring_rx_s%u_io%u_w%u",
 205                                 socket_io,
 206                                 lcore,
 207                                 lcore_worker);
 208                         ring = rte_ring_create(
 209                                 name,
 210                                 app.ring_rx_size,
 211                                 socket_io,
 212                                 RING_F_SP_ENQ | RING_F_SC_DEQ);
 213                         if (ring == NULL) {
 214                                 rte_panic("Cannot create ring to connect I/O core %u with worker core %u\n",
 215                                         lcore,
 216                                         lcore_worker);
 217                         }
 218
 219                         lp_io->rx.rings[lp_io->rx.n_rings] = ring;
 220                         lp_io->rx.n_rings ++;
 221
 222                         lp_worker->rings_in[lp_worker->n_rings_in] = ring;
 223                         lp_worker->n_rings_in ++;
 224                 }
 225         }
 226
 227         for (lcore = 0; lcore < APP_MAX_LCORES; lcore ++) {
 228                 struct app_lcore_params_io *lp_io = &app.lcore_params[lcore].io;
 229
 230                 if ((app.lcore_params[lcore].type != e_APP_LCORE_IO) ||
 231                     (lp_io->rx.n_nic_queues == 0)) {
 232                         continue;
 233                 }
 234
 235                 if (lp_io->rx.n_rings != app_get_lcores_worker()) {
 236                         rte_panic("Algorithmic error (I/O RX rings)\n");
 237                 }
 238         }
 239
 240         for (lcore = 0; lcore < APP_MAX_LCORES; lcore ++) {
 241                 struct app_lcore_params_worker *lp_worker = &app.lcore_params[lcore].worker;
 242
 243                 if (app.lcore_params[lcore].type != e_APP_LCORE_WORKER) {
 244                         continue;
 245                 }
 246
 247                 if (lp_worker->n_rings_in != app_get_lcores_io_rx()) {
 248                         rte_panic("Algorithmic error (worker input rings)\n");
 249                 }
 250         }
 251 }
 252
 253 static void
 254 app_init_rings_tx(void)
 255 {
 256         unsigned lcore;
 257
 258         /* Initialize the rings for the TX side */
 259         for (lcore = 0; lcore < APP_MAX_LCORES; lcore ++) {
 260                 struct app_lcore_params_worker *lp_worker = &app.lcore_params[lcore].worker;
 261                 unsigned port;
 262
 263                 if (app.lcore_params[lcore].type != e_APP_LCORE_WORKER) {
 264                         continue;
 265                 }
 266
 267                 for (port = 0; port < APP_MAX_NIC_PORTS; port ++) {
 268                         char name[32];
 269                         struct app_lcore_params_io *lp_io = NULL;
 270                         struct rte_ring *ring;
 271                         uint32_t socket_io, lcore_io;
 272
 273                         if (app.nic_tx_port_mask[port] == 0) {
 274                                 continue;
 275                         }
 276
 277                         if (app_get_lcore_for_nic_tx(port, &lcore_io) < 0) {
 278                                 rte_panic("Algorithmic error (no I/O core to handle TX of port %u)\n",
 279                                         port);
 280                         }
 281
 282                         lp_io = &app.lcore_params[lcore_io].io;
 283                         socket_io = rte_lcore_to_socket_id(lcore_io);
 284
 285                         printf("Creating ring to connect worker lcore %u with TX port %u (through I/O lcore %u) (socket %u) ...\n",
 286                                 lcore, port, (unsigned)lcore_io, (unsigned)socket_io);
 287                         snprintf(name, sizeof(name), "app_ring_tx_s%u_w%u_p%u", socket_io, lcore, port);
 288                         ring = rte_ring_create(
 289                                 name,
 290                                 app.ring_tx_size,
 291                                 socket_io,
 292                                 RING_F_SP_ENQ | RING_F_SC_DEQ);
 293                         if (ring == NULL) {
 294                                 rte_panic("Cannot create ring to connect worker core %u with TX port %u\n",
 295                                         lcore,
 296                                         port);
 297                         }
 298
 299                         lp_worker->rings_out[port] = ring;
 300                         lp_io->tx.rings[port][lp_worker->worker_id] = ring;
 301                 }
 302         }
 303
 304         for (lcore = 0; lcore < APP_MAX_LCORES; lcore ++) {
 305                 struct app_lcore_params_io *lp_io = &app.lcore_params[lcore].io;
 306                 unsigned i;
 307
 308                 if ((app.lcore_params[lcore].type != e_APP_LCORE_IO) ||
 309                     (lp_io->tx.n_nic_ports == 0)) {
 310                         continue;
 311                 }
 312
 313                 for (i = 0; i < lp_io->tx.n_nic_ports; i ++){
 314                         unsigned port, j;
 315
 316                         port = lp_io->tx.nic_ports[i];
 317                         for (j = 0; j < app_get_lcores_worker(); j ++) {
 318                                 if (lp_io->tx.rings[port][j] == NULL) {
 319                                         rte_panic("Algorithmic error (I/O TX rings)\n");
 320                                 }
 321                         }
 322                 }
 323         }
 324 }
 325
 326 /* Check the link status of all ports in up to 9s, and print them finally */
 327 static void
 328 check_all_ports_link_status(uint16_t port_num, uint32_t port_mask)
 329 {
 330 #define CHECK_INTERVAL 100 /* 100ms */
 331 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
 332         uint16_t portid;
 333         uint8_t count, all_ports_up, print_flag = 0;
 334         struct rte_eth_link link;
 335         uint32_t n_rx_queues, n_tx_queues;
 336
 337         printf("\nChecking link status");
 338         fflush(stdout);
 339         for (count = 0; count <= MAX_CHECK_TIME; count++) {
 340                 all_ports_up = 1;
 341                 for (portid = 0; portid < port_num; portid++) {
 342                         if ((port_mask & (1 << portid)) == 0)
 343                                 continue;
 344                         n_rx_queues = app_get_nic_rx_queues_per_port(portid);
 345                         n_tx_queues = app.nic_tx_port_mask[portid];
 346                         if ((n_rx_queues == 0) && (n_tx_queues == 0))
 347                                 continue;
 348                         memset(&link, 0, sizeof(link));
 349                         rte_eth_link_get_nowait(portid, &link);
 350                         /* print link status if flag set */
 351                         if (print_flag == 1) {
 352                                 if (link.link_status)
 353                                         printf(
 354                                         "Port%d Link Up - speed %uMbps - %s\n",
 355                                                 portid, link.link_speed,
 356                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
 357                                         ("full-duplex") : ("half-duplex\n"));
 358                                 else
 359                                         printf("Port %d Link Down\n", portid);
 360                                 continue;
 361                         }
 362                         /* clear all_ports_up flag if any link down */
 363                         if (link.link_status == ETH_LINK_DOWN) {
 364                                 all_ports_up = 0;
 365                                 break;
 366                         }
 367                 }
 368                 /* after finally printing all link status, get out */
 369                 if (print_flag == 1)
 370                         break;
 371
 372                 if (all_ports_up == 0) {
 373                         printf(".");
 374                         fflush(stdout);
 375                         rte_delay_ms(CHECK_INTERVAL);
 376                 }
 377
 378                 /* set the print_flag if all ports up or timeout */
 379                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
 380                         print_flag = 1;
 381                         printf("done\n");
 382                 }
 383         }
 384 }
 385
 386 static void
 387 app_init_nics(void)
 388 {
 389         unsigned socket;
 390         uint32_t lcore;
 391         uint16_t port;
 392         uint8_t queue;
 393         int ret;
 394         uint32_t n_rx_queues, n_tx_queues;
 395
 396         /* Init NIC ports and queues, then start the ports */
 397         for (port = 0; port < APP_MAX_NIC_PORTS; port ++) {
 398                 struct rte_mempool *pool;
 399                 uint16_t nic_rx_ring_size;
 400                 uint16_t nic_tx_ring_size;
 401                 struct rte_eth_rxconf rxq_conf;
 402                 struct rte_eth_txconf txq_conf;
 403                 struct rte_eth_dev_info dev_info;
 404                 struct rte_eth_conf local_port_conf = port_conf;
 405
 406                 n_rx_queues = app_get_nic_rx_queues_per_port(port);
 407                 n_tx_queues = app.nic_tx_port_mask[port];
 408
 409                 if ((n_rx_queues == 0) && (n_tx_queues == 0)) {
 410                         continue;
 411                 }
 412
 413                 /* Init port */
 414                 printf("Initializing NIC port %u ...\n", port);
 415                 rte_eth_dev_info_get(port, &dev_info);
 416                 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
 417                         local_port_conf.txmode.offloads |=
 418                                 DEV_TX_OFFLOAD_MBUF_FAST_FREE;
 419
 420                 local_port_conf.rx_adv_conf.rss_conf.rss_hf &=
 421                         dev_info.flow_type_rss_offloads;
 422                 if (local_port_conf.rx_adv_conf.rss_conf.rss_hf !=
 423                                 port_conf.rx_adv_conf.rss_conf.rss_hf) {
 424                         printf("Port %u modified RSS hash function based on hardware support,"
 425                                 "requested:%#"PRIx64" configured:%#"PRIx64"\n",
 426                                 port,
 427                                 port_conf.rx_adv_conf.rss_conf.rss_hf,
 428                                 local_port_conf.rx_adv_conf.rss_conf.rss_hf);
 429                 }
 430
 431                 ret = rte_eth_dev_configure(
 432                         port,
 433                         (uint8_t) n_rx_queues,
 434                         (uint8_t) n_tx_queues,
 435                         &local_port_conf);
 436                 if (ret < 0) {
 437                         rte_panic("Cannot init NIC port %u (%d)\n", port, ret);
 438                 }
 439                 rte_eth_promiscuous_enable(port);
 440
 441                 nic_rx_ring_size = app.nic_rx_ring_size;
 442                 nic_tx_ring_size = app.nic_tx_ring_size;
 443                 ret = rte_eth_dev_adjust_nb_rx_tx_desc(
 444                         port, &nic_rx_ring_size, &nic_tx_ring_size);
 445                 if (ret < 0) {
 446                         rte_panic("Cannot adjust number of descriptors for port %u (%d)\n",
 447                                   port, ret);
 448                 }
 449                 app.nic_rx_ring_size = nic_rx_ring_size;
 450                 app.nic_tx_ring_size = nic_tx_ring_size;
 451
 452                 rxq_conf = dev_info.default_rxconf;
 453                 rxq_conf.offloads = local_port_conf.rxmode.offloads;
 454                 /* Init RX queues */
 455                 for (queue = 0; queue < APP_MAX_RX_QUEUES_PER_NIC_PORT; queue ++) {
 456                         if (app.nic_rx_queue_mask[port][queue] == 0) {
 457                                 continue;
 458                         }
 459
 460                         app_get_lcore_for_nic_rx(port, queue, &lcore);
 461                         socket = rte_lcore_to_socket_id(lcore);
 462                         pool = app.lcore_params[lcore].pool;
 463
 464                         printf("Initializing NIC port %u RX queue %u ...\n",
 465                                 port, queue);
 466                         ret = rte_eth_rx_queue_setup(
 467                                 port,
 468                                 queue,
 469                                 (uint16_t) app.nic_rx_ring_size,
 470                                 socket,
 471                                 &rxq_conf,
 472                                 pool);
 473                         if (ret < 0) {
 474                                 rte_panic("Cannot init RX queue %u for port %u (%d)\n",
 475                                           queue, port, ret);
 476                         }
 477                 }
 478
 479                 txq_conf = dev_info.default_txconf;
 480                 txq_conf.offloads = local_port_conf.txmode.offloads;
 481                 /* Init TX queues */
 482                 if (app.nic_tx_port_mask[port] == 1) {
 483                         app_get_lcore_for_nic_tx(port, &lcore);
 484                         socket = rte_lcore_to_socket_id(lcore);
 485                         printf("Initializing NIC port %u TX queue 0 ...\n",
 486                                 port);
 487                         ret = rte_eth_tx_queue_setup(
 488                                 port,
 489                                 0,
 490                                 (uint16_t) app.nic_tx_ring_size,
 491                                 socket,
 492                                 &txq_conf);
 493                         if (ret < 0) {
 494                                 rte_panic("Cannot init TX queue 0 for port %d (%d)\n",
 495                                         port,
 496                                         ret);
 497                         }
 498                 }
 499
 500                 /* Start port */
 501                 ret = rte_eth_dev_start(port);
 502                 if (ret < 0) {
 503                         rte_panic("Cannot start port %d (%d)\n", port, ret);
 504                 }
 505         }
 506
 507         check_all_ports_link_status(APP_MAX_NIC_PORTS, (~0x0));
 508 }
 509
 510 void
 511 app_init(void)
 512 {
 513         app_assign_worker_ids();
 514         app_init_mbuf_pools();
 515         app_init_lpm_tables();
 516         app_init_rings_rx();
 517         app_init_rings_tx();
 518         app_init_nics();
 519
 520         printf("Initialization completed.\n");
 521 }