app/test-pmd/tm.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2017 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33 #include <stdio.h>
  34 #include <sys/stat.h>
  35
  36 #include <rte_cycles.h>
  37 #include <rte_mbuf.h>
  38 #include <rte_ethdev.h>
  39 #include <rte_flow.h>
  40 #include <rte_meter.h>
  41 #include <rte_eth_softnic.h>
  42 #include <rte_tm.h>
  43
  44 #include "testpmd.h"
  45
  46 #define SUBPORT_NODES_PER_PORT          1
  47 #define PIPE_NODES_PER_SUBPORT          4096
  48 #define TC_NODES_PER_PIPE                       4
  49 #define QUEUE_NODES_PER_TC                      4
  50
  51 #define NUM_PIPE_NODES                                          \
  52         (SUBPORT_NODES_PER_PORT * PIPE_NODES_PER_SUBPORT)
  53
  54 #define NUM_TC_NODES                                            \
  55         (NUM_PIPE_NODES * TC_NODES_PER_PIPE)
  56
  57 #define ROOT_NODE_ID                            1000000
  58 #define SUBPORT_NODES_START_ID          900000
  59 #define PIPE_NODES_START_ID                     800000
  60 #define TC_NODES_START_ID                       700000
  61
  62 #define STATS_MASK_DEFAULT                                      \
  63         (RTE_TM_STATS_N_PKTS |                                  \
  64         RTE_TM_STATS_N_BYTES |                                  \
  65         RTE_TM_STATS_N_PKTS_GREEN_DROPPED |                     \
  66         RTE_TM_STATS_N_BYTES_GREEN_DROPPED)
  67
  68 #define STATS_MASK_QUEUE                                        \
  69         (STATS_MASK_DEFAULT |                                   \
  70         RTE_TM_STATS_N_PKTS_QUEUED)
  71
  72 #define BYTES_IN_MBPS                           (1000 * 1000 / 8)
  73 #define TOKEN_BUCKET_SIZE                       1000000
  74
  75 /* TM Hierarchy Levels */
  76 enum tm_hierarchy_level {
  77         TM_NODE_LEVEL_PORT = 0,
  78         TM_NODE_LEVEL_SUBPORT,
  79         TM_NODE_LEVEL_PIPE,
  80         TM_NODE_LEVEL_TC,
  81         TM_NODE_LEVEL_QUEUE,
  82         TM_NODE_LEVEL_MAX,
  83 };
  84
  85 struct tm_hierarchy {
  86         /* TM Nodes */
  87         uint32_t root_node_id;
  88         uint32_t subport_node_id[SUBPORT_NODES_PER_PORT];
  89         uint32_t pipe_node_id[SUBPORT_NODES_PER_PORT][PIPE_NODES_PER_SUBPORT];
  90         uint32_t tc_node_id[NUM_PIPE_NODES][TC_NODES_PER_PIPE];
  91         uint32_t queue_node_id[NUM_TC_NODES][QUEUE_NODES_PER_TC];
  92
  93         /* TM Hierarchy Nodes Shaper Rates */
  94         uint32_t root_node_shaper_rate;
  95         uint32_t subport_node_shaper_rate;
  96         uint32_t pipe_node_shaper_rate;
  97         uint32_t tc_node_shaper_rate;
  98         uint32_t tc_node_shared_shaper_rate;
  99
 100         uint32_t n_shapers;
 101 };
 102
 103 #define BITFIELD(byte_array, slab_pos, slab_mask, slab_shr)     \
 104 ({                                                              \
 105         uint64_t slab = *((uint64_t *) &byte_array[slab_pos]);  \
 106         uint64_t val =                          \
 107                 (rte_be_to_cpu_64(slab) & slab_mask) >> slab_shr;       \
 108         val;                                            \
 109 })
 110
 111 #define RTE_SCHED_PORT_HIERARCHY(subport, pipe,           \
 112         traffic_class, queue, color)                          \
 113         ((((uint64_t) (queue)) & 0x3) |                       \
 114         ((((uint64_t) (traffic_class)) & 0x3) << 2) |         \
 115         ((((uint64_t) (color)) & 0x3) << 4) |                 \
 116         ((((uint64_t) (subport)) & 0xFFFF) << 16) |           \
 117         ((((uint64_t) (pipe)) & 0xFFFFFFFF) << 32))
 118
 119
 120 static void
 121 pkt_metadata_set(struct rte_port *p, struct rte_mbuf **pkts,
 122         uint32_t n_pkts)
 123 {
 124         struct softnic_port_tm *tm = &p->softport.tm;
 125         uint32_t i;
 126
 127         for (i = 0; i < (n_pkts & (~0x3)); i += 4) {
 128                 struct rte_mbuf *pkt0 = pkts[i];
 129                 struct rte_mbuf *pkt1 = pkts[i + 1];
 130                 struct rte_mbuf *pkt2 = pkts[i + 2];
 131                 struct rte_mbuf *pkt3 = pkts[i + 3];
 132
 133                 uint8_t *pkt0_data = rte_pktmbuf_mtod(pkt0, uint8_t *);
 134                 uint8_t *pkt1_data = rte_pktmbuf_mtod(pkt1, uint8_t *);
 135                 uint8_t *pkt2_data = rte_pktmbuf_mtod(pkt2, uint8_t *);
 136                 uint8_t *pkt3_data = rte_pktmbuf_mtod(pkt3, uint8_t *);
 137
 138                 uint64_t pkt0_subport = BITFIELD(pkt0_data,
 139                                         tm->tm_pktfield0_slabpos,
 140                                         tm->tm_pktfield0_slabmask,
 141                                         tm->tm_pktfield0_slabshr);
 142                 uint64_t pkt0_pipe = BITFIELD(pkt0_data,
 143                                         tm->tm_pktfield1_slabpos,
 144                                         tm->tm_pktfield1_slabmask,
 145                                         tm->tm_pktfield1_slabshr);
 146                 uint64_t pkt0_dscp = BITFIELD(pkt0_data,
 147                                         tm->tm_pktfield2_slabpos,
 148                                         tm->tm_pktfield2_slabmask,
 149                                         tm->tm_pktfield2_slabshr);
 150                 uint32_t pkt0_tc = tm->tm_tc_table[pkt0_dscp & 0x3F] >> 2;
 151                 uint32_t pkt0_tc_q = tm->tm_tc_table[pkt0_dscp & 0x3F] & 0x3;
 152                 uint64_t pkt1_subport = BITFIELD(pkt1_data,
 153                                         tm->tm_pktfield0_slabpos,
 154                                         tm->tm_pktfield0_slabmask,
 155                                         tm->tm_pktfield0_slabshr);
 156                 uint64_t pkt1_pipe = BITFIELD(pkt1_data,
 157                                         tm->tm_pktfield1_slabpos,
 158                                         tm->tm_pktfield1_slabmask,
 159                                         tm->tm_pktfield1_slabshr);
 160                 uint64_t pkt1_dscp = BITFIELD(pkt1_data,
 161                                         tm->tm_pktfield2_slabpos,
 162                                         tm->tm_pktfield2_slabmask,
 163                                         tm->tm_pktfield2_slabshr);
 164                 uint32_t pkt1_tc = tm->tm_tc_table[pkt1_dscp & 0x3F] >> 2;
 165                 uint32_t pkt1_tc_q = tm->tm_tc_table[pkt1_dscp & 0x3F] & 0x3;
 166
 167                 uint64_t pkt2_subport = BITFIELD(pkt2_data,
 168                                         tm->tm_pktfield0_slabpos,
 169                                         tm->tm_pktfield0_slabmask,
 170                                         tm->tm_pktfield0_slabshr);
 171                 uint64_t pkt2_pipe = BITFIELD(pkt2_data,
 172                                         tm->tm_pktfield1_slabpos,
 173                                         tm->tm_pktfield1_slabmask,
 174                                         tm->tm_pktfield1_slabshr);
 175                 uint64_t pkt2_dscp = BITFIELD(pkt2_data,
 176                                         tm->tm_pktfield2_slabpos,
 177                                         tm->tm_pktfield2_slabmask,
 178                                         tm->tm_pktfield2_slabshr);
 179                 uint32_t pkt2_tc = tm->tm_tc_table[pkt2_dscp & 0x3F] >> 2;
 180                 uint32_t pkt2_tc_q = tm->tm_tc_table[pkt2_dscp & 0x3F] & 0x3;
 181
 182                 uint64_t pkt3_subport = BITFIELD(pkt3_data,
 183                                         tm->tm_pktfield0_slabpos,
 184                                         tm->tm_pktfield0_slabmask,
 185                                         tm->tm_pktfield0_slabshr);
 186                 uint64_t pkt3_pipe = BITFIELD(pkt3_data,
 187                                         tm->tm_pktfield1_slabpos,
 188                                         tm->tm_pktfield1_slabmask,
 189                                         tm->tm_pktfield1_slabshr);
 190                 uint64_t pkt3_dscp = BITFIELD(pkt3_data,
 191                                         tm->tm_pktfield2_slabpos,
 192                                         tm->tm_pktfield2_slabmask,
 193                                         tm->tm_pktfield2_slabshr);
 194                 uint32_t pkt3_tc = tm->tm_tc_table[pkt3_dscp & 0x3F] >> 2;
 195                 uint32_t pkt3_tc_q = tm->tm_tc_table[pkt3_dscp & 0x3F] & 0x3;
 196
 197                 uint64_t pkt0_sched = RTE_SCHED_PORT_HIERARCHY(pkt0_subport,
 198                                                 pkt0_pipe,
 199                                                 pkt0_tc,
 200                                                 pkt0_tc_q,
 201                                                 0);
 202                 uint64_t pkt1_sched = RTE_SCHED_PORT_HIERARCHY(pkt1_subport,
 203                                                 pkt1_pipe,
 204                                                 pkt1_tc,
 205                                                 pkt1_tc_q,
 206                                                 0);
 207                 uint64_t pkt2_sched = RTE_SCHED_PORT_HIERARCHY(pkt2_subport,
 208                                                 pkt2_pipe,
 209                                                 pkt2_tc,
 210                                                 pkt2_tc_q,
 211                                                 0);
 212                 uint64_t pkt3_sched = RTE_SCHED_PORT_HIERARCHY(pkt3_subport,
 213                                                 pkt3_pipe,
 214                                                 pkt3_tc,
 215                                                 pkt3_tc_q,
 216                                                 0);
 217
 218                 pkt0->hash.sched.lo = pkt0_sched & 0xFFFFFFFF;
 219                 pkt0->hash.sched.hi = pkt0_sched >> 32;
 220                 pkt1->hash.sched.lo = pkt1_sched & 0xFFFFFFFF;
 221                 pkt1->hash.sched.hi = pkt1_sched >> 32;
 222                 pkt2->hash.sched.lo = pkt2_sched & 0xFFFFFFFF;
 223                 pkt2->hash.sched.hi = pkt2_sched >> 32;
 224                 pkt3->hash.sched.lo = pkt3_sched & 0xFFFFFFFF;
 225                 pkt3->hash.sched.hi = pkt3_sched >> 32;
 226         }
 227
 228         for (; i < n_pkts; i++) {
 229                 struct rte_mbuf *pkt = pkts[i];
 230
 231                 uint8_t *pkt_data = rte_pktmbuf_mtod(pkt, uint8_t *);
 232
 233                 uint64_t pkt_subport = BITFIELD(pkt_data,
 234                                         tm->tm_pktfield0_slabpos,
 235                                         tm->tm_pktfield0_slabmask,
 236                                         tm->tm_pktfield0_slabshr);
 237                 uint64_t pkt_pipe = BITFIELD(pkt_data,
 238                                         tm->tm_pktfield1_slabpos,
 239                                         tm->tm_pktfield1_slabmask,
 240                                         tm->tm_pktfield1_slabshr);
 241                 uint64_t pkt_dscp = BITFIELD(pkt_data,
 242                                         tm->tm_pktfield2_slabpos,
 243                                         tm->tm_pktfield2_slabmask,
 244                                         tm->tm_pktfield2_slabshr);
 245                 uint32_t pkt_tc = tm->tm_tc_table[pkt_dscp & 0x3F] >> 2;
 246                 uint32_t pkt_tc_q = tm->tm_tc_table[pkt_dscp & 0x3F] & 0x3;
 247
 248                 uint64_t pkt_sched = RTE_SCHED_PORT_HIERARCHY(pkt_subport,
 249                                                 pkt_pipe,
 250                                                 pkt_tc,
 251                                                 pkt_tc_q,
 252                                                 0);
 253
 254                 pkt->hash.sched.lo = pkt_sched & 0xFFFFFFFF;
 255                 pkt->hash.sched.hi = pkt_sched >> 32;
 256         }
 257 }
 258
 259 /*
 260  * Soft port packet forward
 261  */
 262 static void
 263 softport_packet_fwd(struct fwd_stream *fs)
 264 {
 265         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
 266         struct rte_port *rte_tx_port = &ports[fs->tx_port];
 267         uint16_t nb_rx;
 268         uint16_t nb_tx;
 269         uint32_t retry;
 270
 271 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
 272         uint64_t start_tsc;
 273         uint64_t end_tsc;
 274         uint64_t core_cycles;
 275 #endif
 276
 277 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
 278         start_tsc = rte_rdtsc();
 279 #endif
 280
 281         /*  Packets Receive */
 282         nb_rx = rte_eth_rx_burst(fs->rx_port, fs->rx_queue,
 283                         pkts_burst, nb_pkt_per_burst);
 284         fs->rx_packets += nb_rx;
 285
 286 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 287         fs->rx_burst_stats.pkt_burst_spread[nb_rx]++;
 288 #endif
 289
 290         if (rte_tx_port->softnic_enable) {
 291                 /* Set packet metadata if tm flag enabled */
 292                 if (rte_tx_port->softport.tm_flag)
 293                         pkt_metadata_set(rte_tx_port, pkts_burst, nb_rx);
 294
 295                 /* Softport run */
 296                 rte_pmd_softnic_run(fs->tx_port);
 297         }
 298         nb_tx = rte_eth_tx_burst(fs->tx_port, fs->tx_queue,
 299                         pkts_burst, nb_rx);
 300
 301         /* Retry if necessary */
 302         if (unlikely(nb_tx < nb_rx) && fs->retry_enabled) {
 303                 retry = 0;
 304                 while (nb_tx < nb_rx && retry++ < burst_tx_retry_num) {
 305                         rte_delay_us(burst_tx_delay_time);
 306                         nb_tx += rte_eth_tx_burst(fs->tx_port, fs->tx_queue,
 307                                         &pkts_burst[nb_tx], nb_rx - nb_tx);
 308                 }
 309         }
 310         fs->tx_packets += nb_tx;
 311
 312 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 313         fs->tx_burst_stats.pkt_burst_spread[nb_tx]++;
 314 #endif
 315
 316         if (unlikely(nb_tx < nb_rx)) {
 317                 fs->fwd_dropped += (nb_rx - nb_tx);
 318                 do {
 319                         rte_pktmbuf_free(pkts_burst[nb_tx]);
 320                 } while (++nb_tx < nb_rx);
 321         }
 322 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
 323         end_tsc = rte_rdtsc();
 324         core_cycles = (end_tsc - start_tsc);
 325         fs->core_cycles = (uint64_t) (fs->core_cycles + core_cycles);
 326 #endif
 327 }
 328
 329 static void
 330 set_tm_hiearchy_nodes_shaper_rate(portid_t port_id, struct tm_hierarchy *h)
 331 {
 332         struct rte_eth_link link_params;
 333         uint64_t tm_port_rate;
 334
 335         memset(&link_params, 0, sizeof(link_params));
 336
 337         rte_eth_link_get(port_id, &link_params);
 338         tm_port_rate = (uint64_t)link_params.link_speed * BYTES_IN_MBPS;
 339
 340         if (tm_port_rate > UINT32_MAX)
 341                 tm_port_rate = UINT32_MAX;
 342
 343         /* Set tm hierarchy shapers rate */
 344         h->root_node_shaper_rate = tm_port_rate;
 345         h->subport_node_shaper_rate =
 346                 tm_port_rate / SUBPORT_NODES_PER_PORT;
 347         h->pipe_node_shaper_rate
 348                 = h->subport_node_shaper_rate / PIPE_NODES_PER_SUBPORT;
 349         h->tc_node_shaper_rate = h->pipe_node_shaper_rate;
 350         h->tc_node_shared_shaper_rate = h->subport_node_shaper_rate;
 351 }
 352
 353 static int
 354 softport_tm_root_node_add(portid_t port_id, struct tm_hierarchy *h,
 355         struct rte_tm_error *error)
 356 {
 357         struct rte_tm_node_params rnp;
 358         struct rte_tm_shaper_params rsp;
 359         uint32_t priority, weight, level_id, shaper_profile_id;
 360
 361         memset(&rsp, 0, sizeof(struct rte_tm_shaper_params));
 362         memset(&rnp, 0, sizeof(struct rte_tm_node_params));
 363
 364         /* Shaper profile Parameters */
 365         rsp.peak.rate = h->root_node_shaper_rate;
 366         rsp.peak.size = TOKEN_BUCKET_SIZE;
 367         rsp.pkt_length_adjust = RTE_TM_ETH_FRAMING_OVERHEAD_FCS;
 368         shaper_profile_id = 0;
 369
 370         if (rte_tm_shaper_profile_add(port_id, shaper_profile_id,
 371                 &rsp, error)) {
 372                 printf("%s ERROR(%d)-%s!(shaper_id %u)\n ",
 373                         __func__, error->type, error->message,
 374                         shaper_profile_id);
 375                 return -1;
 376         }
 377
 378         /* Root Node Parameters */
 379         h->root_node_id = ROOT_NODE_ID;
 380         weight = 1;
 381         priority = 0;
 382         level_id = TM_NODE_LEVEL_PORT;
 383         rnp.shaper_profile_id = shaper_profile_id;
 384         rnp.nonleaf.n_sp_priorities = 1;
 385         rnp.stats_mask = STATS_MASK_DEFAULT;
 386
 387         /* Add Node to TM Hierarchy */
 388         if (rte_tm_node_add(port_id, h->root_node_id, RTE_TM_NODE_ID_NULL,
 389                 priority, weight, level_id, &rnp, error)) {
 390                 printf("%s ERROR(%d)-%s!(node_id %u, parent_id %u, level %u)\n",
 391                         __func__, error->type, error->message,
 392                         h->root_node_id, RTE_TM_NODE_ID_NULL,
 393                         level_id);
 394                 return -1;
 395         }
 396         /* Update */
 397         h->n_shapers++;
 398
 399         printf("  Root node added (Start id %u, Count %u, level %u)\n",
 400                 h->root_node_id, 1, level_id);
 401
 402         return 0;
 403 }
 404
 405 static int
 406 softport_tm_subport_node_add(portid_t port_id, struct tm_hierarchy *h,
 407         struct rte_tm_error *error)
 408 {
 409         uint32_t subport_parent_node_id, subport_node_id = 0;
 410         struct rte_tm_node_params snp;
 411         struct rte_tm_shaper_params ssp;
 412         uint32_t priority, weight, level_id, shaper_profile_id;
 413         uint32_t i;
 414
 415         memset(&ssp, 0, sizeof(struct rte_tm_shaper_params));
 416         memset(&snp, 0, sizeof(struct rte_tm_node_params));
 417
 418         shaper_profile_id = h->n_shapers;
 419
 420         /* Add Shaper Profile to TM Hierarchy */
 421         for (i = 0; i < SUBPORT_NODES_PER_PORT; i++) {
 422                 ssp.peak.rate = h->subport_node_shaper_rate;
 423                 ssp.peak.size = TOKEN_BUCKET_SIZE;
 424                 ssp.pkt_length_adjust = RTE_TM_ETH_FRAMING_OVERHEAD_FCS;
 425
 426                 if (rte_tm_shaper_profile_add(port_id, shaper_profile_id,
 427                         &ssp, error)) {
 428                         printf("%s ERROR(%d)-%s!(shaper_id %u)\n ",
 429                                 __func__, error->type, error->message,
 430                                 shaper_profile_id);
 431                         return -1;
 432                 }
 433
 434                 /* Node Parameters */
 435                 h->subport_node_id[i] = SUBPORT_NODES_START_ID + i;
 436                 subport_parent_node_id = h->root_node_id;
 437                 weight = 1;
 438                 priority = 0;
 439                 level_id = TM_NODE_LEVEL_SUBPORT;
 440                 snp.shaper_profile_id = shaper_profile_id;
 441                 snp.nonleaf.n_sp_priorities = 1;
 442                 snp.stats_mask = STATS_MASK_DEFAULT;
 443
 444                 /* Add Node to TM Hiearchy */
 445                 if (rte_tm_node_add(port_id,
 446                                 h->subport_node_id[i],
 447                                 subport_parent_node_id,
 448                                 priority, weight,
 449                                 level_id,
 450                                 &snp,
 451                                 error)) {
 452                         printf("%s ERROR(%d)-%s!(node %u,parent %u,level %u)\n",
 453                                         __func__,
 454                                         error->type,
 455                                         error->message,
 456                                         h->subport_node_id[i],
 457                                         subport_parent_node_id,
 458                                         level_id);
 459                         return -1;
 460                 }
 461                 shaper_profile_id++;
 462                 subport_node_id++;
 463         }
 464         /* Update */
 465         h->n_shapers = shaper_profile_id;
 466
 467         printf("  Subport nodes added (Start id %u, Count %u, level %u)\n",
 468                 h->subport_node_id[0], SUBPORT_NODES_PER_PORT, level_id);
 469
 470         return 0;
 471 }
 472
 473 static int
 474 softport_tm_pipe_node_add(portid_t port_id, struct tm_hierarchy *h,
 475         struct rte_tm_error *error)
 476 {
 477         uint32_t pipe_parent_node_id;
 478         struct rte_tm_node_params pnp;
 479         struct rte_tm_shaper_params psp;
 480         uint32_t priority, weight, level_id, shaper_profile_id;
 481         uint32_t i, j;
 482
 483         memset(&psp, 0, sizeof(struct rte_tm_shaper_params));
 484         memset(&pnp, 0, sizeof(struct rte_tm_node_params));
 485
 486         shaper_profile_id = h->n_shapers;
 487
 488         /* Shaper Profile Parameters */
 489         psp.peak.rate = h->pipe_node_shaper_rate;
 490         psp.peak.size = TOKEN_BUCKET_SIZE;
 491         psp.pkt_length_adjust = RTE_TM_ETH_FRAMING_OVERHEAD_FCS;
 492
 493         /* Pipe Node Parameters */
 494         weight = 1;
 495         priority = 0;
 496         level_id = TM_NODE_LEVEL_PIPE;
 497         pnp.nonleaf.n_sp_priorities = 4;
 498         pnp.stats_mask = STATS_MASK_DEFAULT;
 499
 500         /* Add Shaper Profiles and Nodes to TM Hierarchy */
 501         for (i = 0; i < SUBPORT_NODES_PER_PORT; i++) {
 502                 for (j = 0; j < PIPE_NODES_PER_SUBPORT; j++) {
 503                         if (rte_tm_shaper_profile_add(port_id,
 504                                 shaper_profile_id, &psp, error)) {
 505                                 printf("%s ERROR(%d)-%s!(shaper_id %u)\n ",
 506                                         __func__, error->type, error->message,
 507                                         shaper_profile_id);
 508                                 return -1;
 509                         }
 510                         pnp.shaper_profile_id = shaper_profile_id;
 511                         pipe_parent_node_id = h->subport_node_id[i];
 512                         h->pipe_node_id[i][j] = PIPE_NODES_START_ID +
 513                                 (i * PIPE_NODES_PER_SUBPORT) + j;
 514
 515                         if (rte_tm_node_add(port_id,
 516                                         h->pipe_node_id[i][j],
 517                                         pipe_parent_node_id,
 518                                         priority, weight, level_id,
 519                                         &pnp,
 520                                         error)) {
 521                                 printf("%s ERROR(%d)-%s!(node %u,parent %u )\n",
 522                                         __func__,
 523                                         error->type,
 524                                         error->message,
 525                                         h->pipe_node_id[i][j],
 526                                         pipe_parent_node_id);
 527
 528                                 return -1;
 529                         }
 530                         shaper_profile_id++;
 531                 }
 532         }
 533         /* Update */
 534         h->n_shapers = shaper_profile_id;
 535
 536         printf("  Pipe nodes added (Start id %u, Count %u, level %u)\n",
 537                 h->pipe_node_id[0][0], NUM_PIPE_NODES, level_id);
 538
 539         return 0;
 540 }
 541
 542 static int
 543 softport_tm_tc_node_add(portid_t port_id, struct tm_hierarchy *h,
 544         struct rte_tm_error *error)
 545 {
 546         uint32_t tc_parent_node_id;
 547         struct rte_tm_node_params tnp;
 548         struct rte_tm_shaper_params tsp, tssp;
 549         uint32_t shared_shaper_profile_id[TC_NODES_PER_PIPE];
 550         uint32_t priority, weight, level_id, shaper_profile_id;
 551         uint32_t pos, n_tc_nodes, i, j, k;
 552
 553         memset(&tsp, 0, sizeof(struct rte_tm_shaper_params));
 554         memset(&tssp, 0, sizeof(struct rte_tm_shaper_params));
 555         memset(&tnp, 0, sizeof(struct rte_tm_node_params));
 556
 557         shaper_profile_id = h->n_shapers;
 558
 559         /* Private Shaper Profile (TC) Parameters */
 560         tsp.peak.rate = h->tc_node_shaper_rate;
 561         tsp.peak.size = TOKEN_BUCKET_SIZE;
 562         tsp.pkt_length_adjust = RTE_TM_ETH_FRAMING_OVERHEAD_FCS;
 563
 564         /* Shared Shaper Profile (TC) Parameters */
 565         tssp.peak.rate = h->tc_node_shared_shaper_rate;
 566         tssp.peak.size = TOKEN_BUCKET_SIZE;
 567         tssp.pkt_length_adjust = RTE_TM_ETH_FRAMING_OVERHEAD_FCS;
 568
 569         /* TC Node Parameters */
 570         weight = 1;
 571         level_id = TM_NODE_LEVEL_TC;
 572         tnp.n_shared_shapers = 1;
 573         tnp.nonleaf.n_sp_priorities = 1;
 574         tnp.stats_mask = STATS_MASK_DEFAULT;
 575
 576         /* Add Shared Shaper Profiles to TM Hierarchy */
 577         for (i = 0; i < TC_NODES_PER_PIPE; i++) {
 578                 shared_shaper_profile_id[i] = shaper_profile_id;
 579
 580                 if (rte_tm_shaper_profile_add(port_id,
 581                         shared_shaper_profile_id[i], &tssp, error)) {
 582                         printf("%s ERROR(%d)-%s!(Shared shaper profileid %u)\n",
 583                                 __func__, error->type, error->message,
 584                                 shared_shaper_profile_id[i]);
 585
 586                         return -1;
 587                 }
 588                 if (rte_tm_shared_shaper_add_update(port_id,  i,
 589                         shared_shaper_profile_id[i], error)) {
 590                         printf("%s ERROR(%d)-%s!(Shared shaper id %u)\n",
 591                                 __func__, error->type, error->message, i);
 592
 593                         return -1;
 594                 }
 595                 shaper_profile_id++;
 596         }
 597
 598         /* Add Shaper Profiles and Nodes to TM Hierarchy */
 599         n_tc_nodes = 0;
 600         for (i = 0; i < SUBPORT_NODES_PER_PORT; i++) {
 601                 for (j = 0; j < PIPE_NODES_PER_SUBPORT; j++) {
 602                         for (k = 0; k < TC_NODES_PER_PIPE ; k++) {
 603                                 priority = k;
 604                                 tc_parent_node_id = h->pipe_node_id[i][j];
 605                                 tnp.shared_shaper_id =
 606                                         (uint32_t *)calloc(1, sizeof(uint32_t));
 607                                 tnp.shared_shaper_id[0] = k;
 608                                 pos = j + (i * PIPE_NODES_PER_SUBPORT);
 609                                 h->tc_node_id[pos][k] =
 610                                         TC_NODES_START_ID + n_tc_nodes;
 611
 612                                 if (rte_tm_shaper_profile_add(port_id,
 613                                         shaper_profile_id, &tsp, error)) {
 614                                         printf("%s ERROR(%d)-%s!(shaper %u)\n",
 615                                                 __func__, error->type,
 616                                                 error->message,
 617                                                 shaper_profile_id);
 618
 619                                         free(tnp.shared_shaper_id);
 620                                         return -1;
 621                                 }
 622                                 tnp.shaper_profile_id = shaper_profile_id;
 623                                 if (rte_tm_node_add(port_id,
 624                                                 h->tc_node_id[pos][k],
 625                                                 tc_parent_node_id,
 626                                                 priority, weight,
 627                                                 level_id,
 628                                                 &tnp, error)) {
 629                                         printf("%s ERROR(%d)-%s!(node id %u)\n",
 630                                                 __func__,
 631                                                 error->type,
 632                                                 error->message,
 633                                                 h->tc_node_id[pos][k]);
 634
 635                                         free(tnp.shared_shaper_id);
 636                                         return -1;
 637                                 }
 638                                 shaper_profile_id++;
 639                                 n_tc_nodes++;
 640                         }
 641                 }
 642         }
 643         /* Update */
 644         h->n_shapers = shaper_profile_id;
 645
 646         printf("  TC nodes added (Start id %u, Count %u, level %u)\n",
 647                 h->tc_node_id[0][0], n_tc_nodes, level_id);
 648
 649         return 0;
 650 }
 651
 652 static int
 653 softport_tm_queue_node_add(portid_t port_id, struct tm_hierarchy *h,
 654         struct rte_tm_error *error)
 655 {
 656         uint32_t queue_parent_node_id;
 657         struct rte_tm_node_params qnp;
 658         uint32_t priority, weight, level_id, pos;
 659         uint32_t n_queue_nodes, i, j, k;
 660
 661         memset(&qnp, 0, sizeof(struct rte_tm_node_params));
 662
 663         /* Queue Node Parameters */
 664         priority = 0;
 665         weight = 1;
 666         level_id = TM_NODE_LEVEL_QUEUE;
 667         qnp.shaper_profile_id = RTE_TM_SHAPER_PROFILE_ID_NONE;
 668         qnp.leaf.cman = RTE_TM_CMAN_TAIL_DROP;
 669         qnp.stats_mask = STATS_MASK_QUEUE;
 670
 671         /* Add Queue Nodes to TM Hierarchy */
 672         n_queue_nodes = 0;
 673         for (i = 0; i < NUM_PIPE_NODES; i++) {
 674                 for (j = 0; j < TC_NODES_PER_PIPE; j++) {
 675                         queue_parent_node_id = h->tc_node_id[i][j];
 676                         for (k = 0; k < QUEUE_NODES_PER_TC; k++) {
 677                                 pos = j + (i * TC_NODES_PER_PIPE);
 678                                 h->queue_node_id[pos][k] = n_queue_nodes;
 679                                 if (rte_tm_node_add(port_id,
 680                                                 h->queue_node_id[pos][k],
 681                                                 queue_parent_node_id,
 682                                                 priority,
 683                                                 weight,
 684                                                 level_id,
 685                                                 &qnp, error)) {
 686                                         printf("%s ERROR(%d)-%s!(node %u)\n",
 687                                                 __func__,
 688                                                 error->type,
 689                                                 error->message,
 690                                                 h->queue_node_id[pos][k]);
 691
 692                                         return -1;
 693                                 }
 694                                 n_queue_nodes++;
 695                         }
 696                 }
 697         }
 698         printf("  Queue nodes added (Start id %u, Count %u, level %u)\n",
 699                 h->queue_node_id[0][0], n_queue_nodes, level_id);
 700
 701         return 0;
 702 }
 703
 704 /*
 705  * TM Packet Field Setup
 706  */
 707 static void
 708 softport_tm_pktfield_setup(portid_t port_id)
 709 {
 710         struct rte_port *p = &ports[port_id];
 711         uint64_t pktfield0_mask = 0;
 712         uint64_t pktfield1_mask = 0x0000000FFF000000LLU;
 713         uint64_t pktfield2_mask = 0x00000000000000FCLLU;
 714
 715         p->softport.tm = (struct softnic_port_tm) {
 716                 .n_subports_per_port = SUBPORT_NODES_PER_PORT,
 717                 .n_pipes_per_subport = PIPE_NODES_PER_SUBPORT,
 718
 719                 /* Packet field to identify subport
 720                  *
 721                  * Default configuration assumes only one subport, thus
 722                  * the subport ID is hardcoded to 0
 723                  */
 724                 .tm_pktfield0_slabpos = 0,
 725                 .tm_pktfield0_slabmask = pktfield0_mask,
 726                 .tm_pktfield0_slabshr =
 727                         __builtin_ctzll(pktfield0_mask),
 728
 729                 /* Packet field to identify pipe.
 730                  *
 731                  * Default value assumes Ethernet/IPv4/UDP packets,
 732                  * UDP payload bits 12 .. 23
 733                  */
 734                 .tm_pktfield1_slabpos = 40,
 735                 .tm_pktfield1_slabmask = pktfield1_mask,
 736                 .tm_pktfield1_slabshr =
 737                         __builtin_ctzll(pktfield1_mask),
 738
 739                 /* Packet field used as index into TC translation table
 740                  * to identify the traffic class and queue.
 741                  *
 742                  * Default value assumes Ethernet/IPv4 packets, IPv4
 743                  * DSCP field
 744                  */
 745                 .tm_pktfield2_slabpos = 8,
 746                 .tm_pktfield2_slabmask = pktfield2_mask,
 747                 .tm_pktfield2_slabshr =
 748                         __builtin_ctzll(pktfield2_mask),
 749
 750                 .tm_tc_table = {
 751                         0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
 752                         0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
 753                         0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
 754                         0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
 755                 }, /**< TC translation table */
 756         };
 757 }
 758
 759 static int
 760 softport_tm_hierarchy_specify(portid_t port_id, struct rte_tm_error *error)
 761 {
 762
 763         struct tm_hierarchy h;
 764         int status;
 765
 766         memset(&h, 0, sizeof(struct tm_hierarchy));
 767
 768         /* TM hierarchy shapers rate */
 769         set_tm_hiearchy_nodes_shaper_rate(port_id, &h);
 770
 771         /* Add root node (level 0) */
 772         status = softport_tm_root_node_add(port_id, &h, error);
 773         if (status)
 774                 return status;
 775
 776         /* Add subport node (level 1) */
 777         status = softport_tm_subport_node_add(port_id, &h, error);
 778         if (status)
 779                 return status;
 780
 781         /* Add pipe nodes (level 2) */
 782         status = softport_tm_pipe_node_add(port_id, &h, error);
 783         if (status)
 784                 return status;
 785
 786         /* Add traffic class nodes (level 3) */
 787         status = softport_tm_tc_node_add(port_id, &h, error);
 788         if (status)
 789                 return status;
 790
 791         /* Add queue nodes (level 4) */
 792         status = softport_tm_queue_node_add(port_id, &h, error);
 793         if (status)
 794                 return status;
 795
 796         /* TM packet fields setup */
 797         softport_tm_pktfield_setup(port_id);
 798
 799         return 0;
 800 }
 801
 802 /*
 803  * Soft port Init
 804  */
 805 static void
 806 softport_tm_begin(portid_t pi)
 807 {
 808         struct rte_port *port = &ports[pi];
 809
 810         /* Soft port TM flag */
 811         if (port->softport.tm_flag == 1) {
 812                 printf("\n\n  TM feature available on port %u\n", pi);
 813
 814                 /* Soft port TM hierarchy configuration */
 815                 if ((port->softport.tm.hierarchy_config == 0) &&
 816                         (port->softport.tm.default_hierarchy_enable == 1)) {
 817                         struct rte_tm_error error;
 818                         int status;
 819
 820                         /* Stop port */
 821                         rte_eth_dev_stop(pi);
 822
 823                         /* TM hierarchy specification */
 824                         status = softport_tm_hierarchy_specify(pi, &error);
 825                         if (status) {
 826                                 printf("  TM Hierarchy built error(%d) - %s\n",
 827                                         error.type, error.message);
 828                                 return;
 829                         }
 830                         printf("\n  TM Hierarchy Specified!\n\v");
 831
 832                         /* TM hierarchy commit */
 833                         status = rte_tm_hierarchy_commit(pi, 0, &error);
 834                         if (status) {
 835                                 printf("  Hierarchy commit error(%d) - %s\n",
 836                                         error.type, error.message);
 837                                 return;
 838                         }
 839                         printf("  Hierarchy Committed (port %u)!", pi);
 840                         port->softport.tm.hierarchy_config = 1;
 841
 842                         /* Start port */
 843                         status = rte_eth_dev_start(pi);
 844                         if (status) {
 845                                 printf("\n  Port %u start error!\n", pi);
 846                                 return;
 847                         }
 848                         printf("\n  Port %u started!\n", pi);
 849                         return;
 850                 }
 851         }
 852         printf("\n  TM feature not available on port %u", pi);
 853 }
 854
 855 struct fwd_engine softnic_tm_engine = {
 856         .fwd_mode_name  = "tm",
 857         .port_fwd_begin = softport_tm_begin,
 858         .port_fwd_end   = NULL,
 859         .packet_fwd     = softport_packet_fwd,
 860 };
 861
 862 struct fwd_engine softnic_tm_bypass_engine = {
 863         .fwd_mode_name  = "tm-bypass",
 864         .port_fwd_begin = NULL,
 865         .port_fwd_end   = NULL,
 866         .packet_fwd     = softport_packet_fwd,
 867 };