lib/librte_sched/rte_sched.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2014 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <string.h>
   7
   8 #include <rte_common.h>
   9 #include <rte_log.h>
  10 #include <rte_memory.h>
  11 #include <rte_malloc.h>
  12 #include <rte_cycles.h>
  13 #include <rte_prefetch.h>
  14 #include <rte_branch_prediction.h>
  15 #include <rte_mbuf.h>
  16 #include <rte_bitmap.h>
  17 #include <rte_reciprocal.h>
  18
  19 #include "rte_sched.h"
  20 #include "rte_sched_common.h"
  21 #include "rte_approx.h"
  22
  23 #ifdef __INTEL_COMPILER
  24 #pragma warning(disable:2259) /* conversion may lose significant bits */
  25 #endif
  26
  27 #ifdef RTE_SCHED_VECTOR
  28 #include <rte_vect.h>
  29
  30 #ifdef RTE_ARCH_X86
  31 #define SCHED_VECTOR_SSE4
  32 #elif defined(RTE_MACHINE_CPUFLAG_NEON)
  33 #define SCHED_VECTOR_NEON
  34 #endif
  35
  36 #endif
  37
  38 #define RTE_SCHED_TB_RATE_CONFIG_ERR          (1e-7)
  39 #define RTE_SCHED_WRR_SHIFT                   3
  40 #define RTE_SCHED_GRINDER_PCACHE_SIZE         (64 / RTE_SCHED_QUEUES_PER_PIPE)
  41 #define RTE_SCHED_PIPE_INVALID                UINT32_MAX
  42 #define RTE_SCHED_BMP_POS_INVALID             UINT32_MAX
  43
  44 /* Scaling for cycles_per_byte calculation
  45  * Chosen so that minimum rate is 480 bit/sec
  46  */
  47 #define RTE_SCHED_TIME_SHIFT                  8
  48
  49 struct rte_sched_subport {
  50         /* Token bucket (TB) */
  51         uint64_t tb_time; /* time of last update */
  52         uint32_t tb_period;
  53         uint32_t tb_credits_per_period;
  54         uint32_t tb_size;
  55         uint32_t tb_credits;
  56
  57         /* Traffic classes (TCs) */
  58         uint64_t tc_time; /* time of next update */
  59         uint32_t tc_credits_per_period[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
  60         uint32_t tc_credits[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
  61         uint32_t tc_period;
  62
  63         /* TC oversubscription */
  64         uint32_t tc_ov_wm;
  65         uint32_t tc_ov_wm_min;
  66         uint32_t tc_ov_wm_max;
  67         uint8_t tc_ov_period_id;
  68         uint8_t tc_ov;
  69         uint32_t tc_ov_n;
  70         double tc_ov_rate;
  71
  72         /* Statistics */
  73         struct rte_sched_subport_stats stats;
  74 };
  75
  76 struct rte_sched_pipe_profile {
  77         /* Token bucket (TB) */
  78         uint32_t tb_period;
  79         uint32_t tb_credits_per_period;
  80         uint32_t tb_size;
  81
  82         /* Pipe traffic classes */
  83         uint32_t tc_period;
  84         uint32_t tc_credits_per_period[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
  85         uint8_t tc_ov_weight;
  86
  87         /* Pipe queues */
  88         uint8_t  wrr_cost[RTE_SCHED_QUEUES_PER_PIPE];
  89 };
  90
  91 struct rte_sched_pipe {
  92         /* Token bucket (TB) */
  93         uint64_t tb_time; /* time of last update */
  94         uint32_t tb_credits;
  95
  96         /* Pipe profile and flags */
  97         uint32_t profile;
  98
  99         /* Traffic classes (TCs) */
 100         uint64_t tc_time; /* time of next update */
 101         uint32_t tc_credits[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
 102
 103         /* Weighted Round Robin (WRR) */
 104         uint8_t wrr_tokens[RTE_SCHED_QUEUES_PER_PIPE];
 105
 106         /* TC oversubscription */
 107         uint32_t tc_ov_credits;
 108         uint8_t tc_ov_period_id;
 109         uint8_t reserved[3];
 110 } __rte_cache_aligned;
 111
 112 struct rte_sched_queue {
 113         uint16_t qw;
 114         uint16_t qr;
 115 };
 116
 117 struct rte_sched_queue_extra {
 118         struct rte_sched_queue_stats stats;
 119 #ifdef RTE_SCHED_RED
 120         struct rte_red red;
 121 #endif
 122 };
 123
 124 enum grinder_state {
 125         e_GRINDER_PREFETCH_PIPE = 0,
 126         e_GRINDER_PREFETCH_TC_QUEUE_ARRAYS,
 127         e_GRINDER_PREFETCH_MBUF,
 128         e_GRINDER_READ_MBUF
 129 };
 130
 131 /*
 132  * Path through the scheduler hierarchy used by the scheduler enqueue
 133  * operation to identify the destination queue for the current
 134  * packet. Stored in the field pkt.hash.sched of struct rte_mbuf of
 135  * each packet, typically written by the classification stage and read
 136  * by scheduler enqueue.
 137  */
 138 struct rte_sched_port_hierarchy {
 139         uint16_t queue:2;                /**< Queue ID (0 .. 3) */
 140         uint16_t traffic_class:2;        /**< Traffic class ID (0 .. 3)*/
 141         uint32_t color:2;                /**< Color */
 142         uint16_t unused:10;
 143         uint16_t subport;                /**< Subport ID */
 144         uint32_t pipe;                   /**< Pipe ID */
 145 };
 146
 147 struct rte_sched_grinder {
 148         /* Pipe cache */
 149         uint16_t pcache_qmask[RTE_SCHED_GRINDER_PCACHE_SIZE];
 150         uint32_t pcache_qindex[RTE_SCHED_GRINDER_PCACHE_SIZE];
 151         uint32_t pcache_w;
 152         uint32_t pcache_r;
 153
 154         /* Current pipe */
 155         enum grinder_state state;
 156         uint32_t productive;
 157         uint32_t pindex;
 158         struct rte_sched_subport *subport;
 159         struct rte_sched_pipe *pipe;
 160         struct rte_sched_pipe_profile *pipe_params;
 161
 162         /* TC cache */
 163         uint8_t tccache_qmask[4];
 164         uint32_t tccache_qindex[4];
 165         uint32_t tccache_w;
 166         uint32_t tccache_r;
 167
 168         /* Current TC */
 169         uint32_t tc_index;
 170         struct rte_sched_queue *queue[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
 171         struct rte_mbuf **qbase[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
 172         uint32_t qindex[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
 173         uint16_t qsize;
 174         uint32_t qmask;
 175         uint32_t qpos;
 176         struct rte_mbuf *pkt;
 177
 178         /* WRR */
 179         uint16_t wrr_tokens[RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS];
 180         uint16_t wrr_mask[RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS];
 181         uint8_t wrr_cost[RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS];
 182 };
 183
 184 struct rte_sched_port {
 185         /* User parameters */
 186         uint32_t n_subports_per_port;
 187         uint32_t n_pipes_per_subport;
 188         uint32_t rate;
 189         uint32_t mtu;
 190         uint32_t frame_overhead;
 191         uint16_t qsize[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
 192         uint32_t n_pipe_profiles;
 193         uint32_t pipe_tc3_rate_max;
 194 #ifdef RTE_SCHED_RED
 195         struct rte_red_config red_config[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE][e_RTE_METER_COLORS];
 196 #endif
 197
 198         /* Timing */
 199         uint64_t time_cpu_cycles;     /* Current CPU time measured in CPU cyles */
 200         uint64_t time_cpu_bytes;      /* Current CPU time measured in bytes */
 201         uint64_t time;                /* Current NIC TX time measured in bytes */
 202         struct rte_reciprocal inv_cycles_per_byte; /* CPU cycles per byte */
 203
 204         /* Scheduling loop detection */
 205         uint32_t pipe_loop;
 206         uint32_t pipe_exhaustion;
 207
 208         /* Bitmap */
 209         struct rte_bitmap *bmp;
 210         uint32_t grinder_base_bmp_pos[RTE_SCHED_PORT_N_GRINDERS] __rte_aligned_16;
 211
 212         /* Grinders */
 213         struct rte_sched_grinder grinder[RTE_SCHED_PORT_N_GRINDERS];
 214         uint32_t busy_grinders;
 215         struct rte_mbuf **pkts_out;
 216         uint32_t n_pkts_out;
 217
 218         /* Queue base calculation */
 219         uint32_t qsize_add[RTE_SCHED_QUEUES_PER_PIPE];
 220         uint32_t qsize_sum;
 221
 222         /* Large data structures */
 223         struct rte_sched_subport *subport;
 224         struct rte_sched_pipe *pipe;
 225         struct rte_sched_queue *queue;
 226         struct rte_sched_queue_extra *queue_extra;
 227         struct rte_sched_pipe_profile *pipe_profiles;
 228         uint8_t *bmp_array;
 229         struct rte_mbuf **queue_array;
 230         uint8_t memory[0] __rte_cache_aligned;
 231 } __rte_cache_aligned;
 232
 233 enum rte_sched_port_array {
 234         e_RTE_SCHED_PORT_ARRAY_SUBPORT = 0,
 235         e_RTE_SCHED_PORT_ARRAY_PIPE,
 236         e_RTE_SCHED_PORT_ARRAY_QUEUE,
 237         e_RTE_SCHED_PORT_ARRAY_QUEUE_EXTRA,
 238         e_RTE_SCHED_PORT_ARRAY_PIPE_PROFILES,
 239         e_RTE_SCHED_PORT_ARRAY_BMP_ARRAY,
 240         e_RTE_SCHED_PORT_ARRAY_QUEUE_ARRAY,
 241         e_RTE_SCHED_PORT_ARRAY_TOTAL,
 242 };
 243
 244 #ifdef RTE_SCHED_COLLECT_STATS
 245
 246 static inline uint32_t
 247 rte_sched_port_queues_per_subport(struct rte_sched_port *port)
 248 {
 249         return RTE_SCHED_QUEUES_PER_PIPE * port->n_pipes_per_subport;
 250 }
 251
 252 #endif
 253
 254 static inline uint32_t
 255 rte_sched_port_queues_per_port(struct rte_sched_port *port)
 256 {
 257         return RTE_SCHED_QUEUES_PER_PIPE * port->n_pipes_per_subport * port->n_subports_per_port;
 258 }
 259
 260 static inline struct rte_mbuf **
 261 rte_sched_port_qbase(struct rte_sched_port *port, uint32_t qindex)
 262 {
 263         uint32_t pindex = qindex >> 4;
 264         uint32_t qpos = qindex & 0xF;
 265
 266         return (port->queue_array + pindex *
 267                 port->qsize_sum + port->qsize_add[qpos]);
 268 }
 269
 270 static inline uint16_t
 271 rte_sched_port_qsize(struct rte_sched_port *port, uint32_t qindex)
 272 {
 273         uint32_t tc = (qindex >> 2) & 0x3;
 274
 275         return port->qsize[tc];
 276 }
 277
 278 static int
 279 pipe_profile_check(struct rte_sched_pipe_params *params,
 280         uint32_t rate)
 281 {
 282         uint32_t i;
 283
 284         /* Pipe parameters */
 285         if (params == NULL)
 286                 return -10;
 287
 288         /* TB rate: non-zero, not greater than port rate */
 289         if (params->tb_rate == 0 ||
 290                 params->tb_rate > rate)
 291                 return -11;
 292
 293         /* TB size: non-zero */
 294         if (params->tb_size == 0)
 295                 return -12;
 296
 297         /* TC rate: non-zero, less than pipe rate */
 298         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++) {
 299                 if (params->tc_rate[i] == 0 ||
 300                         params->tc_rate[i] > params->tb_rate)
 301                         return -13;
 302         }
 303
 304         /* TC period: non-zero */
 305         if (params->tc_period == 0)
 306                 return -14;
 307
 308 #ifdef RTE_SCHED_SUBPORT_TC_OV
 309         /* TC3 oversubscription weight: non-zero */
 310         if (params->tc_ov_weight == 0)
 311                 return -15;
 312 #endif
 313
 314         /* Queue WRR weights: non-zero */
 315         for (i = 0; i < RTE_SCHED_QUEUES_PER_PIPE; i++) {
 316                 if (params->wrr_weights[i] == 0)
 317                         return -16;
 318         }
 319
 320         return 0;
 321 }
 322
 323 static int
 324 rte_sched_port_check_params(struct rte_sched_port_params *params)
 325 {
 326         uint32_t i;
 327
 328         if (params == NULL)
 329                 return -1;
 330
 331         /* socket */
 332         if ((params->socket < 0) || (params->socket >= RTE_MAX_NUMA_NODES))
 333                 return -3;
 334
 335         /* rate */
 336         if (params->rate == 0)
 337                 return -4;
 338
 339         /* mtu */
 340         if (params->mtu == 0)
 341                 return -5;
 342
 343         /* n_subports_per_port: non-zero, limited to 16 bits, power of 2 */
 344         if (params->n_subports_per_port == 0 ||
 345             params->n_subports_per_port > 1u << 16 ||
 346             !rte_is_power_of_2(params->n_subports_per_port))
 347                 return -6;
 348
 349         /* n_pipes_per_subport: non-zero, power of 2 */
 350         if (params->n_pipes_per_subport == 0 ||
 351             !rte_is_power_of_2(params->n_pipes_per_subport))
 352                 return -7;
 353
 354         /* qsize: non-zero, power of 2,
 355          * no bigger than 32K (due to 16-bit read/write pointers)
 356          */
 357         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++) {
 358                 uint16_t qsize = params->qsize[i];
 359
 360                 if (qsize == 0 || !rte_is_power_of_2(qsize))
 361                         return -8;
 362         }
 363
 364         /* pipe_profiles and n_pipe_profiles */
 365         if (params->pipe_profiles == NULL ||
 366             params->n_pipe_profiles == 0 ||
 367             params->n_pipe_profiles > RTE_SCHED_PIPE_PROFILES_PER_PORT)
 368                 return -9;
 369
 370         for (i = 0; i < params->n_pipe_profiles; i++) {
 371                 struct rte_sched_pipe_params *p = params->pipe_profiles + i;
 372                 int status;
 373
 374                 status = pipe_profile_check(p, params->rate);
 375                 if (status != 0)
 376                         return status;
 377         }
 378
 379         return 0;
 380 }
 381
 382 static uint32_t
 383 rte_sched_port_get_array_base(struct rte_sched_port_params *params, enum rte_sched_port_array array)
 384 {
 385         uint32_t n_subports_per_port = params->n_subports_per_port;
 386         uint32_t n_pipes_per_subport = params->n_pipes_per_subport;
 387         uint32_t n_pipes_per_port = n_pipes_per_subport * n_subports_per_port;
 388         uint32_t n_queues_per_port = RTE_SCHED_QUEUES_PER_PIPE * n_pipes_per_subport * n_subports_per_port;
 389
 390         uint32_t size_subport = n_subports_per_port * sizeof(struct rte_sched_subport);
 391         uint32_t size_pipe = n_pipes_per_port * sizeof(struct rte_sched_pipe);
 392         uint32_t size_queue = n_queues_per_port * sizeof(struct rte_sched_queue);
 393         uint32_t size_queue_extra
 394                 = n_queues_per_port * sizeof(struct rte_sched_queue_extra);
 395         uint32_t size_pipe_profiles
 396                 = RTE_SCHED_PIPE_PROFILES_PER_PORT * sizeof(struct rte_sched_pipe_profile);
 397         uint32_t size_bmp_array = rte_bitmap_get_memory_footprint(n_queues_per_port);
 398         uint32_t size_per_pipe_queue_array, size_queue_array;
 399
 400         uint32_t base, i;
 401
 402         size_per_pipe_queue_array = 0;
 403         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++) {
 404                 size_per_pipe_queue_array += RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS
 405                         * params->qsize[i] * sizeof(struct rte_mbuf *);
 406         }
 407         size_queue_array = n_pipes_per_port * size_per_pipe_queue_array;
 408
 409         base = 0;
 410
 411         if (array == e_RTE_SCHED_PORT_ARRAY_SUBPORT)
 412                 return base;
 413         base += RTE_CACHE_LINE_ROUNDUP(size_subport);
 414
 415         if (array == e_RTE_SCHED_PORT_ARRAY_PIPE)
 416                 return base;
 417         base += RTE_CACHE_LINE_ROUNDUP(size_pipe);
 418
 419         if (array == e_RTE_SCHED_PORT_ARRAY_QUEUE)
 420                 return base;
 421         base += RTE_CACHE_LINE_ROUNDUP(size_queue);
 422
 423         if (array == e_RTE_SCHED_PORT_ARRAY_QUEUE_EXTRA)
 424                 return base;
 425         base += RTE_CACHE_LINE_ROUNDUP(size_queue_extra);
 426
 427         if (array == e_RTE_SCHED_PORT_ARRAY_PIPE_PROFILES)
 428                 return base;
 429         base += RTE_CACHE_LINE_ROUNDUP(size_pipe_profiles);
 430
 431         if (array == e_RTE_SCHED_PORT_ARRAY_BMP_ARRAY)
 432                 return base;
 433         base += RTE_CACHE_LINE_ROUNDUP(size_bmp_array);
 434
 435         if (array == e_RTE_SCHED_PORT_ARRAY_QUEUE_ARRAY)
 436                 return base;
 437         base += RTE_CACHE_LINE_ROUNDUP(size_queue_array);
 438
 439         return base;
 440 }
 441
 442 uint32_t
 443 rte_sched_port_get_memory_footprint(struct rte_sched_port_params *params)
 444 {
 445         uint32_t size0, size1;
 446         int status;
 447
 448         status = rte_sched_port_check_params(params);
 449         if (status != 0) {
 450                 RTE_LOG(NOTICE, SCHED,
 451                         "Port scheduler params check failed (%d)\n", status);
 452
 453                 return 0;
 454         }
 455
 456         size0 = sizeof(struct rte_sched_port);
 457         size1 = rte_sched_port_get_array_base(params, e_RTE_SCHED_PORT_ARRAY_TOTAL);
 458
 459         return size0 + size1;
 460 }
 461
 462 static void
 463 rte_sched_port_config_qsize(struct rte_sched_port *port)
 464 {
 465         /* TC 0 */
 466         port->qsize_add[0] = 0;
 467         port->qsize_add[1] = port->qsize_add[0] + port->qsize[0];
 468         port->qsize_add[2] = port->qsize_add[1] + port->qsize[0];
 469         port->qsize_add[3] = port->qsize_add[2] + port->qsize[0];
 470
 471         /* TC 1 */
 472         port->qsize_add[4] = port->qsize_add[3] + port->qsize[0];
 473         port->qsize_add[5] = port->qsize_add[4] + port->qsize[1];
 474         port->qsize_add[6] = port->qsize_add[5] + port->qsize[1];
 475         port->qsize_add[7] = port->qsize_add[6] + port->qsize[1];
 476
 477         /* TC 2 */
 478         port->qsize_add[8] = port->qsize_add[7] + port->qsize[1];
 479         port->qsize_add[9] = port->qsize_add[8] + port->qsize[2];
 480         port->qsize_add[10] = port->qsize_add[9] + port->qsize[2];
 481         port->qsize_add[11] = port->qsize_add[10] + port->qsize[2];
 482
 483         /* TC 3 */
 484         port->qsize_add[12] = port->qsize_add[11] + port->qsize[2];
 485         port->qsize_add[13] = port->qsize_add[12] + port->qsize[3];
 486         port->qsize_add[14] = port->qsize_add[13] + port->qsize[3];
 487         port->qsize_add[15] = port->qsize_add[14] + port->qsize[3];
 488
 489         port->qsize_sum = port->qsize_add[15] + port->qsize[3];
 490 }
 491
 492 static void
 493 rte_sched_port_log_pipe_profile(struct rte_sched_port *port, uint32_t i)
 494 {
 495         struct rte_sched_pipe_profile *p = port->pipe_profiles + i;
 496
 497         RTE_LOG(DEBUG, SCHED, "Low level config for pipe profile %u:\n"
 498                 "    Token bucket: period = %u, credits per period = %u, size = %u\n"
 499                 "    Traffic classes: period = %u, credits per period = [%u, %u, %u, %u]\n"
 500                 "    Traffic class 3 oversubscription: weight = %hhu\n"
 501                 "    WRR cost: [%hhu, %hhu, %hhu, %hhu], [%hhu, %hhu, %hhu, %hhu], [%hhu, %hhu, %hhu, %hhu], [%hhu, %hhu, %hhu, %hhu]\n",
 502                 i,
 503
 504                 /* Token bucket */
 505                 p->tb_period,
 506                 p->tb_credits_per_period,
 507                 p->tb_size,
 508
 509                 /* Traffic classes */
 510                 p->tc_period,
 511                 p->tc_credits_per_period[0],
 512                 p->tc_credits_per_period[1],
 513                 p->tc_credits_per_period[2],
 514                 p->tc_credits_per_period[3],
 515
 516                 /* Traffic class 3 oversubscription */
 517                 p->tc_ov_weight,
 518
 519                 /* WRR */
 520                 p->wrr_cost[ 0], p->wrr_cost[ 1], p->wrr_cost[ 2], p->wrr_cost[ 3],
 521                 p->wrr_cost[ 4], p->wrr_cost[ 5], p->wrr_cost[ 6], p->wrr_cost[ 7],
 522                 p->wrr_cost[ 8], p->wrr_cost[ 9], p->wrr_cost[10], p->wrr_cost[11],
 523                 p->wrr_cost[12], p->wrr_cost[13], p->wrr_cost[14], p->wrr_cost[15]);
 524 }
 525
 526 static inline uint64_t
 527 rte_sched_time_ms_to_bytes(uint32_t time_ms, uint32_t rate)
 528 {
 529         uint64_t time = time_ms;
 530
 531         time = (time * rate) / 1000;
 532
 533         return time;
 534 }
 535
 536 static void
 537 rte_sched_pipe_profile_convert(struct rte_sched_pipe_params *src,
 538         struct rte_sched_pipe_profile *dst,
 539         uint32_t rate)
 540 {
 541         uint32_t i;
 542
 543         /* Token Bucket */
 544         if (src->tb_rate == rate) {
 545                 dst->tb_credits_per_period = 1;
 546                 dst->tb_period = 1;
 547         } else {
 548                 double tb_rate = (double) src->tb_rate
 549                                 / (double) rate;
 550                 double d = RTE_SCHED_TB_RATE_CONFIG_ERR;
 551
 552                 rte_approx(tb_rate, d,
 553                         &dst->tb_credits_per_period, &dst->tb_period);
 554         }
 555
 556         dst->tb_size = src->tb_size;
 557
 558         /* Traffic Classes */
 559         dst->tc_period = rte_sched_time_ms_to_bytes(src->tc_period,
 560                                                 rate);
 561
 562         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++)
 563                 dst->tc_credits_per_period[i]
 564                         = rte_sched_time_ms_to_bytes(src->tc_period,
 565                                 src->tc_rate[i]);
 566
 567 #ifdef RTE_SCHED_SUBPORT_TC_OV
 568         dst->tc_ov_weight = src->tc_ov_weight;
 569 #endif
 570
 571         /* WRR */
 572         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++) {
 573                 uint32_t wrr_cost[RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS];
 574                 uint32_t lcd, lcd1, lcd2;
 575                 uint32_t qindex;
 576
 577                 qindex = i * RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS;
 578
 579                 wrr_cost[0] = src->wrr_weights[qindex];
 580                 wrr_cost[1] = src->wrr_weights[qindex + 1];
 581                 wrr_cost[2] = src->wrr_weights[qindex + 2];
 582                 wrr_cost[3] = src->wrr_weights[qindex + 3];
 583
 584                 lcd1 = rte_get_lcd(wrr_cost[0], wrr_cost[1]);
 585                 lcd2 = rte_get_lcd(wrr_cost[2], wrr_cost[3]);
 586                 lcd = rte_get_lcd(lcd1, lcd2);
 587
 588                 wrr_cost[0] = lcd / wrr_cost[0];
 589                 wrr_cost[1] = lcd / wrr_cost[1];
 590                 wrr_cost[2] = lcd / wrr_cost[2];
 591                 wrr_cost[3] = lcd / wrr_cost[3];
 592
 593                 dst->wrr_cost[qindex] = (uint8_t) wrr_cost[0];
 594                 dst->wrr_cost[qindex + 1] = (uint8_t) wrr_cost[1];
 595                 dst->wrr_cost[qindex + 2] = (uint8_t) wrr_cost[2];
 596                 dst->wrr_cost[qindex + 3] = (uint8_t) wrr_cost[3];
 597         }
 598 }
 599
 600 static void
 601 rte_sched_port_config_pipe_profile_table(struct rte_sched_port *port,
 602         struct rte_sched_port_params *params)
 603 {
 604         uint32_t i;
 605
 606         for (i = 0; i < port->n_pipe_profiles; i++) {
 607                 struct rte_sched_pipe_params *src = params->pipe_profiles + i;
 608                 struct rte_sched_pipe_profile *dst = port->pipe_profiles + i;
 609
 610                 rte_sched_pipe_profile_convert(src, dst, params->rate);
 611                 rte_sched_port_log_pipe_profile(port, i);
 612         }
 613
 614         port->pipe_tc3_rate_max = 0;
 615         for (i = 0; i < port->n_pipe_profiles; i++) {
 616                 struct rte_sched_pipe_params *src = params->pipe_profiles + i;
 617                 uint32_t pipe_tc3_rate = src->tc_rate[3];
 618
 619                 if (port->pipe_tc3_rate_max < pipe_tc3_rate)
 620                         port->pipe_tc3_rate_max = pipe_tc3_rate;
 621         }
 622 }
 623
 624 struct rte_sched_port *
 625 rte_sched_port_config(struct rte_sched_port_params *params)
 626 {
 627         struct rte_sched_port *port = NULL;
 628         uint32_t mem_size, bmp_mem_size, n_queues_per_port, i, cycles_per_byte;
 629
 630         /* Check user parameters. Determine the amount of memory to allocate */
 631         mem_size = rte_sched_port_get_memory_footprint(params);
 632         if (mem_size == 0)
 633                 return NULL;
 634
 635         /* Allocate memory to store the data structures */
 636         port = rte_zmalloc("qos_params", mem_size, RTE_CACHE_LINE_SIZE);
 637         if (port == NULL)
 638                 return NULL;
 639
 640         /* compile time checks */
 641         RTE_BUILD_BUG_ON(RTE_SCHED_PORT_N_GRINDERS == 0);
 642         RTE_BUILD_BUG_ON(RTE_SCHED_PORT_N_GRINDERS & (RTE_SCHED_PORT_N_GRINDERS - 1));
 643
 644         /* User parameters */
 645         port->n_subports_per_port = params->n_subports_per_port;
 646         port->n_pipes_per_subport = params->n_pipes_per_subport;
 647         port->rate = params->rate;
 648         port->mtu = params->mtu + params->frame_overhead;
 649         port->frame_overhead = params->frame_overhead;
 650         memcpy(port->qsize, params->qsize, sizeof(params->qsize));
 651         port->n_pipe_profiles = params->n_pipe_profiles;
 652
 653 #ifdef RTE_SCHED_RED
 654         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++) {
 655                 uint32_t j;
 656
 657                 for (j = 0; j < e_RTE_METER_COLORS; j++) {
 658                         /* if min/max are both zero, then RED is disabled */
 659                         if ((params->red_params[i][j].min_th |
 660                              params->red_params[i][j].max_th) == 0) {
 661                                 continue;
 662                         }
 663
 664                         if (rte_red_config_init(&port->red_config[i][j],
 665                                 params->red_params[i][j].wq_log2,
 666                                 params->red_params[i][j].min_th,
 667                                 params->red_params[i][j].max_th,
 668                                 params->red_params[i][j].maxp_inv) != 0) {
 669                                 return NULL;
 670                         }
 671                 }
 672         }
 673 #endif
 674
 675         /* Timing */
 676         port->time_cpu_cycles = rte_get_tsc_cycles();
 677         port->time_cpu_bytes = 0;
 678         port->time = 0;
 679
 680         cycles_per_byte = (rte_get_tsc_hz() << RTE_SCHED_TIME_SHIFT)
 681                 / params->rate;
 682         port->inv_cycles_per_byte = rte_reciprocal_value(cycles_per_byte);
 683
 684         /* Scheduling loop detection */
 685         port->pipe_loop = RTE_SCHED_PIPE_INVALID;
 686         port->pipe_exhaustion = 0;
 687
 688         /* Grinders */
 689         port->busy_grinders = 0;
 690         port->pkts_out = NULL;
 691         port->n_pkts_out = 0;
 692
 693         /* Queue base calculation */
 694         rte_sched_port_config_qsize(port);
 695
 696         /* Large data structures */
 697         port->subport = (struct rte_sched_subport *)
 698                 (port->memory + rte_sched_port_get_array_base(params,
 699                                                               e_RTE_SCHED_PORT_ARRAY_SUBPORT));
 700         port->pipe = (struct rte_sched_pipe *)
 701                 (port->memory + rte_sched_port_get_array_base(params,
 702                                                               e_RTE_SCHED_PORT_ARRAY_PIPE));
 703         port->queue = (struct rte_sched_queue *)
 704                 (port->memory + rte_sched_port_get_array_base(params,
 705                                                               e_RTE_SCHED_PORT_ARRAY_QUEUE));
 706         port->queue_extra = (struct rte_sched_queue_extra *)
 707                 (port->memory + rte_sched_port_get_array_base(params,
 708                                                               e_RTE_SCHED_PORT_ARRAY_QUEUE_EXTRA));
 709         port->pipe_profiles = (struct rte_sched_pipe_profile *)
 710                 (port->memory + rte_sched_port_get_array_base(params,
 711                                                               e_RTE_SCHED_PORT_ARRAY_PIPE_PROFILES));
 712         port->bmp_array =  port->memory
 713                 + rte_sched_port_get_array_base(params, e_RTE_SCHED_PORT_ARRAY_BMP_ARRAY);
 714         port->queue_array = (struct rte_mbuf **)
 715                 (port->memory + rte_sched_port_get_array_base(params,
 716                                                               e_RTE_SCHED_PORT_ARRAY_QUEUE_ARRAY));
 717
 718         /* Pipe profile table */
 719         rte_sched_port_config_pipe_profile_table(port, params);
 720
 721         /* Bitmap */
 722         n_queues_per_port = rte_sched_port_queues_per_port(port);
 723         bmp_mem_size = rte_bitmap_get_memory_footprint(n_queues_per_port);
 724         port->bmp = rte_bitmap_init(n_queues_per_port, port->bmp_array,
 725                                     bmp_mem_size);
 726         if (port->bmp == NULL) {
 727                 RTE_LOG(ERR, SCHED, "Bitmap init error\n");
 728                 return NULL;
 729         }
 730
 731         for (i = 0; i < RTE_SCHED_PORT_N_GRINDERS; i++)
 732                 port->grinder_base_bmp_pos[i] = RTE_SCHED_PIPE_INVALID;
 733
 734
 735         return port;
 736 }
 737
 738 void
 739 rte_sched_port_free(struct rte_sched_port *port)
 740 {
 741         uint32_t qindex;
 742         uint32_t n_queues_per_port;
 743
 744         /* Check user parameters */
 745         if (port == NULL)
 746                 return;
 747
 748         n_queues_per_port = rte_sched_port_queues_per_port(port);
 749
 750         /* Free enqueued mbufs */
 751         for (qindex = 0; qindex < n_queues_per_port; qindex++) {
 752                 struct rte_mbuf **mbufs = rte_sched_port_qbase(port, qindex);
 753                 uint16_t qsize = rte_sched_port_qsize(port, qindex);
 754                 struct rte_sched_queue *queue = port->queue + qindex;
 755                 uint16_t qr = queue->qr & (qsize - 1);
 756                 uint16_t qw = queue->qw & (qsize - 1);
 757
 758                 for (; qr != qw; qr = (qr + 1) & (qsize - 1))
 759                         rte_pktmbuf_free(mbufs[qr]);
 760         }
 761
 762         rte_bitmap_free(port->bmp);
 763         rte_free(port);
 764 }
 765
 766 static void
 767 rte_sched_port_log_subport_config(struct rte_sched_port *port, uint32_t i)
 768 {
 769         struct rte_sched_subport *s = port->subport + i;
 770
 771         RTE_LOG(DEBUG, SCHED, "Low level config for subport %u:\n"
 772                 "    Token bucket: period = %u, credits per period = %u, size = %u\n"
 773                 "    Traffic classes: period = %u, credits per period = [%u, %u, %u, %u]\n"
 774                 "    Traffic class 3 oversubscription: wm min = %u, wm max = %u\n",
 775                 i,
 776
 777                 /* Token bucket */
 778                 s->tb_period,
 779                 s->tb_credits_per_period,
 780                 s->tb_size,
 781
 782                 /* Traffic classes */
 783                 s->tc_period,
 784                 s->tc_credits_per_period[0],
 785                 s->tc_credits_per_period[1],
 786                 s->tc_credits_per_period[2],
 787                 s->tc_credits_per_period[3],
 788
 789                 /* Traffic class 3 oversubscription */
 790                 s->tc_ov_wm_min,
 791                 s->tc_ov_wm_max);
 792 }
 793
 794 int
 795 rte_sched_subport_config(struct rte_sched_port *port,
 796         uint32_t subport_id,
 797         struct rte_sched_subport_params *params)
 798 {
 799         struct rte_sched_subport *s;
 800         uint32_t i;
 801
 802         /* Check user parameters */
 803         if (port == NULL ||
 804             subport_id >= port->n_subports_per_port ||
 805             params == NULL)
 806                 return -1;
 807
 808         if (params->tb_rate == 0 || params->tb_rate > port->rate)
 809                 return -2;
 810
 811         if (params->tb_size == 0)
 812                 return -3;
 813
 814         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++) {
 815                 if (params->tc_rate[i] == 0 ||
 816                     params->tc_rate[i] > params->tb_rate)
 817                         return -4;
 818         }
 819
 820         if (params->tc_period == 0)
 821                 return -5;
 822
 823         s = port->subport + subport_id;
 824
 825         /* Token Bucket (TB) */
 826         if (params->tb_rate == port->rate) {
 827                 s->tb_credits_per_period = 1;
 828                 s->tb_period = 1;
 829         } else {
 830                 double tb_rate = ((double) params->tb_rate) / ((double) port->rate);
 831                 double d = RTE_SCHED_TB_RATE_CONFIG_ERR;
 832
 833                 rte_approx(tb_rate, d, &s->tb_credits_per_period, &s->tb_period);
 834         }
 835
 836         s->tb_size = params->tb_size;
 837         s->tb_time = port->time;
 838         s->tb_credits = s->tb_size / 2;
 839
 840         /* Traffic Classes (TCs) */
 841         s->tc_period = rte_sched_time_ms_to_bytes(params->tc_period, port->rate);
 842         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++) {
 843                 s->tc_credits_per_period[i]
 844                         = rte_sched_time_ms_to_bytes(params->tc_period,
 845                                                      params->tc_rate[i]);
 846         }
 847         s->tc_time = port->time + s->tc_period;
 848         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++)
 849                 s->tc_credits[i] = s->tc_credits_per_period[i];
 850
 851 #ifdef RTE_SCHED_SUBPORT_TC_OV
 852         /* TC oversubscription */
 853         s->tc_ov_wm_min = port->mtu;
 854         s->tc_ov_wm_max = rte_sched_time_ms_to_bytes(params->tc_period,
 855                                                      port->pipe_tc3_rate_max);
 856         s->tc_ov_wm = s->tc_ov_wm_max;
 857         s->tc_ov_period_id = 0;
 858         s->tc_ov = 0;
 859         s->tc_ov_n = 0;
 860         s->tc_ov_rate = 0;
 861 #endif
 862
 863         rte_sched_port_log_subport_config(port, subport_id);
 864
 865         return 0;
 866 }
 867
 868 int
 869 rte_sched_pipe_config(struct rte_sched_port *port,
 870         uint32_t subport_id,
 871         uint32_t pipe_id,
 872         int32_t pipe_profile)
 873 {
 874         struct rte_sched_subport *s;
 875         struct rte_sched_pipe *p;
 876         struct rte_sched_pipe_profile *params;
 877         uint32_t deactivate, profile, i;
 878
 879         /* Check user parameters */
 880         profile = (uint32_t) pipe_profile;
 881         deactivate = (pipe_profile < 0);
 882
 883         if (port == NULL ||
 884             subport_id >= port->n_subports_per_port ||
 885             pipe_id >= port->n_pipes_per_subport ||
 886             (!deactivate && profile >= port->n_pipe_profiles))
 887                 return -1;
 888
 889
 890         /* Check that subport configuration is valid */
 891         s = port->subport + subport_id;
 892         if (s->tb_period == 0)
 893                 return -2;
 894
 895         p = port->pipe + (subport_id * port->n_pipes_per_subport + pipe_id);
 896
 897         /* Handle the case when pipe already has a valid configuration */
 898         if (p->tb_time) {
 899                 params = port->pipe_profiles + p->profile;
 900
 901 #ifdef RTE_SCHED_SUBPORT_TC_OV
 902                 double subport_tc3_rate = (double) s->tc_credits_per_period[3]
 903                         / (double) s->tc_period;
 904                 double pipe_tc3_rate = (double) params->tc_credits_per_period[3]
 905                         / (double) params->tc_period;
 906                 uint32_t tc3_ov = s->tc_ov;
 907
 908                 /* Unplug pipe from its subport */
 909                 s->tc_ov_n -= params->tc_ov_weight;
 910                 s->tc_ov_rate -= pipe_tc3_rate;
 911                 s->tc_ov = s->tc_ov_rate > subport_tc3_rate;
 912
 913                 if (s->tc_ov != tc3_ov) {
 914                         RTE_LOG(DEBUG, SCHED,
 915                                 "Subport %u TC3 oversubscription is OFF (%.4lf >= %.4lf)\n",
 916                                 subport_id, subport_tc3_rate, s->tc_ov_rate);
 917                 }
 918 #endif
 919
 920                 /* Reset the pipe */
 921                 memset(p, 0, sizeof(struct rte_sched_pipe));
 922         }
 923
 924         if (deactivate)
 925                 return 0;
 926
 927         /* Apply the new pipe configuration */
 928         p->profile = profile;
 929         params = port->pipe_profiles + p->profile;
 930
 931         /* Token Bucket (TB) */
 932         p->tb_time = port->time;
 933         p->tb_credits = params->tb_size / 2;
 934
 935         /* Traffic Classes (TCs) */
 936         p->tc_time = port->time + params->tc_period;
 937         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++)
 938                 p->tc_credits[i] = params->tc_credits_per_period[i];
 939
 940 #ifdef RTE_SCHED_SUBPORT_TC_OV
 941         {
 942                 /* Subport TC3 oversubscription */
 943                 double subport_tc3_rate = (double) s->tc_credits_per_period[3]
 944                         / (double) s->tc_period;
 945                 double pipe_tc3_rate = (double) params->tc_credits_per_period[3]
 946                         / (double) params->tc_period;
 947                 uint32_t tc3_ov = s->tc_ov;
 948
 949                 s->tc_ov_n += params->tc_ov_weight;
 950                 s->tc_ov_rate += pipe_tc3_rate;
 951                 s->tc_ov = s->tc_ov_rate > subport_tc3_rate;
 952
 953                 if (s->tc_ov != tc3_ov) {
 954                         RTE_LOG(DEBUG, SCHED,
 955                                 "Subport %u TC3 oversubscription is ON (%.4lf < %.4lf)\n",
 956                                 subport_id, subport_tc3_rate, s->tc_ov_rate);
 957                 }
 958                 p->tc_ov_period_id = s->tc_ov_period_id;
 959                 p->tc_ov_credits = s->tc_ov_wm;
 960         }
 961 #endif
 962
 963         return 0;
 964 }
 965
 966 int __rte_experimental
 967 rte_sched_port_pipe_profile_add(struct rte_sched_port *port,
 968         struct rte_sched_pipe_params *params,
 969         uint32_t *pipe_profile_id)
 970 {
 971         struct rte_sched_pipe_profile *pp;
 972         uint32_t i;
 973         int status;
 974
 975         /* Port */
 976         if (port == NULL)
 977                 return -1;
 978
 979         /* Pipe profiles not exceeds the max limit */
 980         if (port->n_pipe_profiles >= RTE_SCHED_PIPE_PROFILES_PER_PORT)
 981                 return -2;
 982
 983         /* Pipe params */
 984         status = pipe_profile_check(params, port->rate);
 985         if (status != 0)
 986                 return status;
 987
 988         pp = &port->pipe_profiles[port->n_pipe_profiles];
 989         rte_sched_pipe_profile_convert(params, pp, port->rate);
 990
 991         /* Pipe profile not exists */
 992         for (i = 0; i < port->n_pipe_profiles; i++)
 993                 if (memcmp(port->pipe_profiles + i, pp, sizeof(*pp)) == 0)
 994                         return -3;
 995
 996         /* Pipe profile commit */
 997         *pipe_profile_id = port->n_pipe_profiles;
 998         port->n_pipe_profiles++;
 999
1000         if (port->pipe_tc3_rate_max < params->tc_rate[3])
1001                 port->pipe_tc3_rate_max = params->tc_rate[3];
1002
1003         rte_sched_port_log_pipe_profile(port, *pipe_profile_id);
1004
1005         return 0;
1006 }
1007
1008 void
1009 rte_sched_port_pkt_write(struct rte_mbuf *pkt,
1010                          uint32_t subport, uint32_t pipe, uint32_t traffic_class,
1011                          uint32_t queue, enum rte_meter_color color)
1012 {
1013         struct rte_sched_port_hierarchy *sched
1014                 = (struct rte_sched_port_hierarchy *) &pkt->hash.sched;
1015
1016         RTE_BUILD_BUG_ON(sizeof(*sched) > sizeof(pkt->hash.sched));
1017
1018         sched->color = (uint32_t) color;
1019         sched->subport = subport;
1020         sched->pipe = pipe;
1021         sched->traffic_class = traffic_class;
1022         sched->queue = queue;
1023 }
1024
1025 void
1026 rte_sched_port_pkt_read_tree_path(const struct rte_mbuf *pkt,
1027                                   uint32_t *subport, uint32_t *pipe,
1028                                   uint32_t *traffic_class, uint32_t *queue)
1029 {
1030         const struct rte_sched_port_hierarchy *sched
1031                 = (const struct rte_sched_port_hierarchy *) &pkt->hash.sched;
1032
1033         *subport = sched->subport;
1034         *pipe = sched->pipe;
1035         *traffic_class = sched->traffic_class;
1036         *queue = sched->queue;
1037 }
1038
1039 enum rte_meter_color
1040 rte_sched_port_pkt_read_color(const struct rte_mbuf *pkt)
1041 {
1042         const struct rte_sched_port_hierarchy *sched
1043                 = (const struct rte_sched_port_hierarchy *) &pkt->hash.sched;
1044
1045         return (enum rte_meter_color) sched->color;
1046 }
1047
1048 int
1049 rte_sched_subport_read_stats(struct rte_sched_port *port,
1050                              uint32_t subport_id,
1051                              struct rte_sched_subport_stats *stats,
1052                              uint32_t *tc_ov)
1053 {
1054         struct rte_sched_subport *s;
1055
1056         /* Check user parameters */
1057         if (port == NULL || subport_id >= port->n_subports_per_port ||
1058             stats == NULL || tc_ov == NULL)
1059                 return -1;
1060
1061         s = port->subport + subport_id;
1062
1063         /* Copy subport stats and clear */
1064         memcpy(stats, &s->stats, sizeof(struct rte_sched_subport_stats));
1065         memset(&s->stats, 0, sizeof(struct rte_sched_subport_stats));
1066
1067         /* Subport TC oversubscription status */
1068         *tc_ov = s->tc_ov;
1069
1070         return 0;
1071 }
1072
1073 int
1074 rte_sched_queue_read_stats(struct rte_sched_port *port,
1075         uint32_t queue_id,
1076         struct rte_sched_queue_stats *stats,
1077         uint16_t *qlen)
1078 {
1079         struct rte_sched_queue *q;
1080         struct rte_sched_queue_extra *qe;
1081
1082         /* Check user parameters */
1083         if ((port == NULL) ||
1084             (queue_id >= rte_sched_port_queues_per_port(port)) ||
1085                 (stats == NULL) ||
1086                 (qlen == NULL)) {
1087                 return -1;
1088         }
1089         q = port->queue + queue_id;
1090         qe = port->queue_extra + queue_id;
1091
1092         /* Copy queue stats and clear */
1093         memcpy(stats, &qe->stats, sizeof(struct rte_sched_queue_stats));
1094         memset(&qe->stats, 0, sizeof(struct rte_sched_queue_stats));
1095
1096         /* Queue length */
1097         *qlen = q->qw - q->qr;
1098
1099         return 0;
1100 }
1101
1102 static inline uint32_t
1103 rte_sched_port_qindex(struct rte_sched_port *port, uint32_t subport, uint32_t pipe, uint32_t traffic_class, uint32_t queue)
1104 {
1105         uint32_t result;
1106
1107         result = subport * port->n_pipes_per_subport + pipe;
1108         result = result * RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE + traffic_class;
1109         result = result * RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS + queue;
1110
1111         return result;
1112 }
1113
1114 #ifdef RTE_SCHED_DEBUG
1115
1116 static inline int
1117 rte_sched_port_queue_is_empty(struct rte_sched_port *port, uint32_t qindex)
1118 {
1119         struct rte_sched_queue *queue = port->queue + qindex;
1120
1121         return queue->qr == queue->qw;
1122 }
1123
1124 #endif /* RTE_SCHED_DEBUG */
1125
1126 #ifdef RTE_SCHED_COLLECT_STATS
1127
1128 static inline void
1129 rte_sched_port_update_subport_stats(struct rte_sched_port *port, uint32_t qindex, struct rte_mbuf *pkt)
1130 {
1131         struct rte_sched_subport *s = port->subport + (qindex / rte_sched_port_queues_per_subport(port));
1132         uint32_t tc_index = (qindex >> 2) & 0x3;
1133         uint32_t pkt_len = pkt->pkt_len;
1134
1135         s->stats.n_pkts_tc[tc_index] += 1;
1136         s->stats.n_bytes_tc[tc_index] += pkt_len;
1137 }
1138
1139 #ifdef RTE_SCHED_RED
1140 static inline void
1141 rte_sched_port_update_subport_stats_on_drop(struct rte_sched_port *port,
1142                                                 uint32_t qindex,
1143                                                 struct rte_mbuf *pkt, uint32_t red)
1144 #else
1145 static inline void
1146 rte_sched_port_update_subport_stats_on_drop(struct rte_sched_port *port,
1147                                                 uint32_t qindex,
1148                                                 struct rte_mbuf *pkt, __rte_unused uint32_t red)
1149 #endif
1150 {
1151         struct rte_sched_subport *s = port->subport + (qindex / rte_sched_port_queues_per_subport(port));
1152         uint32_t tc_index = (qindex >> 2) & 0x3;
1153         uint32_t pkt_len = pkt->pkt_len;
1154
1155         s->stats.n_pkts_tc_dropped[tc_index] += 1;
1156         s->stats.n_bytes_tc_dropped[tc_index] += pkt_len;
1157 #ifdef RTE_SCHED_RED
1158         s->stats.n_pkts_red_dropped[tc_index] += red;
1159 #endif
1160 }
1161
1162 static inline void
1163 rte_sched_port_update_queue_stats(struct rte_sched_port *port, uint32_t qindex, struct rte_mbuf *pkt)
1164 {
1165         struct rte_sched_queue_extra *qe = port->queue_extra + qindex;
1166         uint32_t pkt_len = pkt->pkt_len;
1167
1168         qe->stats.n_pkts += 1;
1169         qe->stats.n_bytes += pkt_len;
1170 }
1171
1172 #ifdef RTE_SCHED_RED
1173 static inline void
1174 rte_sched_port_update_queue_stats_on_drop(struct rte_sched_port *port,
1175                                                 uint32_t qindex,
1176                                                 struct rte_mbuf *pkt, uint32_t red)
1177 #else
1178 static inline void
1179 rte_sched_port_update_queue_stats_on_drop(struct rte_sched_port *port,
1180                                                 uint32_t qindex,
1181                                                 struct rte_mbuf *pkt, __rte_unused uint32_t red)
1182 #endif
1183 {
1184         struct rte_sched_queue_extra *qe = port->queue_extra + qindex;
1185         uint32_t pkt_len = pkt->pkt_len;
1186
1187         qe->stats.n_pkts_dropped += 1;
1188         qe->stats.n_bytes_dropped += pkt_len;
1189 #ifdef RTE_SCHED_RED
1190         qe->stats.n_pkts_red_dropped += red;
1191 #endif
1192 }
1193
1194 #endif /* RTE_SCHED_COLLECT_STATS */
1195
1196 #ifdef RTE_SCHED_RED
1197
1198 static inline int
1199 rte_sched_port_red_drop(struct rte_sched_port *port, struct rte_mbuf *pkt, uint32_t qindex, uint16_t qlen)
1200 {
1201         struct rte_sched_queue_extra *qe;
1202         struct rte_red_config *red_cfg;
1203         struct rte_red *red;
1204         uint32_t tc_index;
1205         enum rte_meter_color color;
1206
1207         tc_index = (qindex >> 2) & 0x3;
1208         color = rte_sched_port_pkt_read_color(pkt);
1209         red_cfg = &port->red_config[tc_index][color];
1210
1211         if ((red_cfg->min_th | red_cfg->max_th) == 0)
1212                 return 0;
1213
1214         qe = port->queue_extra + qindex;
1215         red = &qe->red;
1216
1217         return rte_red_enqueue(red_cfg, red, qlen, port->time);
1218 }
1219
1220 static inline void
1221 rte_sched_port_set_queue_empty_timestamp(struct rte_sched_port *port, uint32_t qindex)
1222 {
1223         struct rte_sched_queue_extra *qe = port->queue_extra + qindex;
1224         struct rte_red *red = &qe->red;
1225
1226         rte_red_mark_queue_empty(red, port->time);
1227 }
1228
1229 #else
1230
1231 #define rte_sched_port_red_drop(port, pkt, qindex, qlen)             0
1232
1233 #define rte_sched_port_set_queue_empty_timestamp(port, qindex)
1234
1235 #endif /* RTE_SCHED_RED */
1236
1237 #ifdef RTE_SCHED_DEBUG
1238
1239 static inline void
1240 debug_check_queue_slab(struct rte_sched_port *port, uint32_t bmp_pos,
1241                        uint64_t bmp_slab)
1242 {
1243         uint64_t mask;
1244         uint32_t i, panic;
1245
1246         if (bmp_slab == 0)
1247                 rte_panic("Empty slab at position %u\n", bmp_pos);
1248
1249         panic = 0;
1250         for (i = 0, mask = 1; i < 64; i++, mask <<= 1) {
1251                 if (mask & bmp_slab) {
1252                         if (rte_sched_port_queue_is_empty(port, bmp_pos + i)) {
1253                                 printf("Queue %u (slab offset %u) is empty\n", bmp_pos + i, i);
1254                                 panic = 1;
1255                         }
1256                 }
1257         }
1258
1259         if (panic)
1260                 rte_panic("Empty queues in slab 0x%" PRIx64 "starting at position %u\n",
1261                         bmp_slab, bmp_pos);
1262 }
1263
1264 #endif /* RTE_SCHED_DEBUG */
1265
1266 static inline uint32_t
1267 rte_sched_port_enqueue_qptrs_prefetch0(struct rte_sched_port *port,
1268                                        struct rte_mbuf *pkt)
1269 {
1270         struct rte_sched_queue *q;
1271 #ifdef RTE_SCHED_COLLECT_STATS
1272         struct rte_sched_queue_extra *qe;
1273 #endif
1274         uint32_t subport, pipe, traffic_class, queue, qindex;
1275
1276         rte_sched_port_pkt_read_tree_path(pkt, &subport, &pipe, &traffic_class, &queue);
1277
1278         qindex = rte_sched_port_qindex(port, subport, pipe, traffic_class, queue);
1279         q = port->queue + qindex;
1280         rte_prefetch0(q);
1281 #ifdef RTE_SCHED_COLLECT_STATS
1282         qe = port->queue_extra + qindex;
1283         rte_prefetch0(qe);
1284 #endif
1285
1286         return qindex;
1287 }
1288
1289 static inline void
1290 rte_sched_port_enqueue_qwa_prefetch0(struct rte_sched_port *port,
1291                                      uint32_t qindex, struct rte_mbuf **qbase)
1292 {
1293         struct rte_sched_queue *q;
1294         struct rte_mbuf **q_qw;
1295         uint16_t qsize;
1296
1297         q = port->queue + qindex;
1298         qsize = rte_sched_port_qsize(port, qindex);
1299         q_qw = qbase + (q->qw & (qsize - 1));
1300
1301         rte_prefetch0(q_qw);
1302         rte_bitmap_prefetch0(port->bmp, qindex);
1303 }
1304
1305 static inline int
1306 rte_sched_port_enqueue_qwa(struct rte_sched_port *port, uint32_t qindex,
1307                            struct rte_mbuf **qbase, struct rte_mbuf *pkt)
1308 {
1309         struct rte_sched_queue *q;
1310         uint16_t qsize;
1311         uint16_t qlen;
1312
1313         q = port->queue + qindex;
1314         qsize = rte_sched_port_qsize(port, qindex);
1315         qlen = q->qw - q->qr;
1316
1317         /* Drop the packet (and update drop stats) when queue is full */
1318         if (unlikely(rte_sched_port_red_drop(port, pkt, qindex, qlen) ||
1319                      (qlen >= qsize))) {
1320                 rte_pktmbuf_free(pkt);
1321 #ifdef RTE_SCHED_COLLECT_STATS
1322                 rte_sched_port_update_subport_stats_on_drop(port, qindex, pkt,
1323                                                             qlen < qsize);
1324                 rte_sched_port_update_queue_stats_on_drop(port, qindex, pkt,
1325                                                           qlen < qsize);
1326 #endif
1327                 return 0;
1328         }
1329
1330         /* Enqueue packet */
1331         qbase[q->qw & (qsize - 1)] = pkt;
1332         q->qw++;
1333
1334         /* Activate queue in the port bitmap */
1335         rte_bitmap_set(port->bmp, qindex);
1336
1337         /* Statistics */
1338 #ifdef RTE_SCHED_COLLECT_STATS
1339         rte_sched_port_update_subport_stats(port, qindex, pkt);
1340         rte_sched_port_update_queue_stats(port, qindex, pkt);
1341 #endif
1342
1343         return 1;
1344 }
1345
1346
1347 /*
1348  * The enqueue function implements a 4-level pipeline with each stage
1349  * processing two different packets. The purpose of using a pipeline
1350  * is to hide the latency of prefetching the data structures. The
1351  * naming convention is presented in the diagram below:
1352  *
1353  *   p00  _______   p10  _______   p20  _______   p30  _______
1354  * ----->|       |----->|       |----->|       |----->|       |----->
1355  *       |   0   |      |   1   |      |   2   |      |   3   |
1356  * ----->|_______|----->|_______|----->|_______|----->|_______|----->
1357  *   p01            p11            p21            p31
1358  *
1359  */
1360 int
1361 rte_sched_port_enqueue(struct rte_sched_port *port, struct rte_mbuf **pkts,
1362                        uint32_t n_pkts)
1363 {
1364         struct rte_mbuf *pkt00, *pkt01, *pkt10, *pkt11, *pkt20, *pkt21,
1365                 *pkt30, *pkt31, *pkt_last;
1366         struct rte_mbuf **q00_base, **q01_base, **q10_base, **q11_base,
1367                 **q20_base, **q21_base, **q30_base, **q31_base, **q_last_base;
1368         uint32_t q00, q01, q10, q11, q20, q21, q30, q31, q_last;
1369         uint32_t r00, r01, r10, r11, r20, r21, r30, r31, r_last;
1370         uint32_t result, i;
1371
1372         result = 0;
1373
1374         /*
1375          * Less then 6 input packets available, which is not enough to
1376          * feed the pipeline
1377          */
1378         if (unlikely(n_pkts < 6)) {
1379                 struct rte_mbuf **q_base[5];
1380                 uint32_t q[5];
1381
1382                 /* Prefetch the mbuf structure of each packet */
1383                 for (i = 0; i < n_pkts; i++)
1384                         rte_prefetch0(pkts[i]);
1385
1386                 /* Prefetch the queue structure for each queue */
1387                 for (i = 0; i < n_pkts; i++)
1388                         q[i] = rte_sched_port_enqueue_qptrs_prefetch0(port,
1389                                                                       pkts[i]);
1390
1391                 /* Prefetch the write pointer location of each queue */
1392                 for (i = 0; i < n_pkts; i++) {
1393                         q_base[i] = rte_sched_port_qbase(port, q[i]);
1394                         rte_sched_port_enqueue_qwa_prefetch0(port, q[i],
1395                                                              q_base[i]);
1396                 }
1397
1398                 /* Write each packet to its queue */
1399                 for (i = 0; i < n_pkts; i++)
1400                         result += rte_sched_port_enqueue_qwa(port, q[i],
1401                                                              q_base[i], pkts[i]);
1402
1403                 return result;
1404         }
1405
1406         /* Feed the first 3 stages of the pipeline (6 packets needed) */
1407         pkt20 = pkts[0];
1408         pkt21 = pkts[1];
1409         rte_prefetch0(pkt20);
1410         rte_prefetch0(pkt21);
1411
1412         pkt10 = pkts[2];
1413         pkt11 = pkts[3];
1414         rte_prefetch0(pkt10);
1415         rte_prefetch0(pkt11);
1416
1417         q20 = rte_sched_port_enqueue_qptrs_prefetch0(port, pkt20);
1418         q21 = rte_sched_port_enqueue_qptrs_prefetch0(port, pkt21);
1419
1420         pkt00 = pkts[4];
1421         pkt01 = pkts[5];
1422         rte_prefetch0(pkt00);
1423         rte_prefetch0(pkt01);
1424
1425         q10 = rte_sched_port_enqueue_qptrs_prefetch0(port, pkt10);
1426         q11 = rte_sched_port_enqueue_qptrs_prefetch0(port, pkt11);
1427
1428         q20_base = rte_sched_port_qbase(port, q20);
1429         q21_base = rte_sched_port_qbase(port, q21);
1430         rte_sched_port_enqueue_qwa_prefetch0(port, q20, q20_base);
1431         rte_sched_port_enqueue_qwa_prefetch0(port, q21, q21_base);
1432
1433         /* Run the pipeline */
1434         for (i = 6; i < (n_pkts & (~1)); i += 2) {
1435                 /* Propagate stage inputs */
1436                 pkt30 = pkt20;
1437                 pkt31 = pkt21;
1438                 pkt20 = pkt10;
1439                 pkt21 = pkt11;
1440                 pkt10 = pkt00;
1441                 pkt11 = pkt01;
1442                 q30 = q20;
1443                 q31 = q21;
1444                 q20 = q10;
1445                 q21 = q11;
1446                 q30_base = q20_base;
1447                 q31_base = q21_base;
1448
1449                 /* Stage 0: Get packets in */
1450                 pkt00 = pkts[i];
1451                 pkt01 = pkts[i + 1];
1452                 rte_prefetch0(pkt00);
1453                 rte_prefetch0(pkt01);
1454
1455                 /* Stage 1: Prefetch queue structure storing queue pointers */
1456                 q10 = rte_sched_port_enqueue_qptrs_prefetch0(port, pkt10);
1457                 q11 = rte_sched_port_enqueue_qptrs_prefetch0(port, pkt11);
1458
1459                 /* Stage 2: Prefetch queue write location */
1460                 q20_base = rte_sched_port_qbase(port, q20);
1461                 q21_base = rte_sched_port_qbase(port, q21);
1462                 rte_sched_port_enqueue_qwa_prefetch0(port, q20, q20_base);
1463                 rte_sched_port_enqueue_qwa_prefetch0(port, q21, q21_base);
1464
1465                 /* Stage 3: Write packet to queue and activate queue */
1466                 r30 = rte_sched_port_enqueue_qwa(port, q30, q30_base, pkt30);
1467                 r31 = rte_sched_port_enqueue_qwa(port, q31, q31_base, pkt31);
1468                 result += r30 + r31;
1469         }
1470
1471         /*
1472          * Drain the pipeline (exactly 6 packets).
1473          * Handle the last packet in the case
1474          * of an odd number of input packets.
1475          */
1476         pkt_last = pkts[n_pkts - 1];
1477         rte_prefetch0(pkt_last);
1478
1479         q00 = rte_sched_port_enqueue_qptrs_prefetch0(port, pkt00);
1480         q01 = rte_sched_port_enqueue_qptrs_prefetch0(port, pkt01);
1481
1482         q10_base = rte_sched_port_qbase(port, q10);
1483         q11_base = rte_sched_port_qbase(port, q11);
1484         rte_sched_port_enqueue_qwa_prefetch0(port, q10, q10_base);
1485         rte_sched_port_enqueue_qwa_prefetch0(port, q11, q11_base);
1486
1487         r20 = rte_sched_port_enqueue_qwa(port, q20, q20_base, pkt20);
1488         r21 = rte_sched_port_enqueue_qwa(port, q21, q21_base, pkt21);
1489         result += r20 + r21;
1490
1491         q_last = rte_sched_port_enqueue_qptrs_prefetch0(port, pkt_last);
1492
1493         q00_base = rte_sched_port_qbase(port, q00);
1494         q01_base = rte_sched_port_qbase(port, q01);
1495         rte_sched_port_enqueue_qwa_prefetch0(port, q00, q00_base);
1496         rte_sched_port_enqueue_qwa_prefetch0(port, q01, q01_base);
1497
1498         r10 = rte_sched_port_enqueue_qwa(port, q10, q10_base, pkt10);
1499         r11 = rte_sched_port_enqueue_qwa(port, q11, q11_base, pkt11);
1500         result += r10 + r11;
1501
1502         q_last_base = rte_sched_port_qbase(port, q_last);
1503         rte_sched_port_enqueue_qwa_prefetch0(port, q_last, q_last_base);
1504
1505         r00 = rte_sched_port_enqueue_qwa(port, q00, q00_base, pkt00);
1506         r01 = rte_sched_port_enqueue_qwa(port, q01, q01_base, pkt01);
1507         result += r00 + r01;
1508
1509         if (n_pkts & 1) {
1510                 r_last = rte_sched_port_enqueue_qwa(port, q_last, q_last_base, pkt_last);
1511                 result += r_last;
1512         }
1513
1514         return result;
1515 }
1516
1517 #ifndef RTE_SCHED_SUBPORT_TC_OV
1518
1519 static inline void
1520 grinder_credits_update(struct rte_sched_port *port, uint32_t pos)
1521 {
1522         struct rte_sched_grinder *grinder = port->grinder + pos;
1523         struct rte_sched_subport *subport = grinder->subport;
1524         struct rte_sched_pipe *pipe = grinder->pipe;
1525         struct rte_sched_pipe_profile *params = grinder->pipe_params;
1526         uint64_t n_periods;
1527
1528         /* Subport TB */
1529         n_periods = (port->time - subport->tb_time) / subport->tb_period;
1530         subport->tb_credits += n_periods * subport->tb_credits_per_period;
1531         subport->tb_credits = rte_sched_min_val_2_u32(subport->tb_credits, subport->tb_size);
1532         subport->tb_time += n_periods * subport->tb_period;
1533
1534         /* Pipe TB */
1535         n_periods = (port->time - pipe->tb_time) / params->tb_period;
1536         pipe->tb_credits += n_periods * params->tb_credits_per_period;
1537         pipe->tb_credits = rte_sched_min_val_2_u32(pipe->tb_credits, params->tb_size);
1538         pipe->tb_time += n_periods * params->tb_period;
1539
1540         /* Subport TCs */
1541         if (unlikely(port->time >= subport->tc_time)) {
1542                 subport->tc_credits[0] = subport->tc_credits_per_period[0];
1543                 subport->tc_credits[1] = subport->tc_credits_per_period[1];
1544                 subport->tc_credits[2] = subport->tc_credits_per_period[2];
1545                 subport->tc_credits[3] = subport->tc_credits_per_period[3];
1546                 subport->tc_time = port->time + subport->tc_period;
1547         }
1548
1549         /* Pipe TCs */
1550         if (unlikely(port->time >= pipe->tc_time)) {
1551                 pipe->tc_credits[0] = params->tc_credits_per_period[0];
1552                 pipe->tc_credits[1] = params->tc_credits_per_period[1];
1553                 pipe->tc_credits[2] = params->tc_credits_per_period[2];
1554                 pipe->tc_credits[3] = params->tc_credits_per_period[3];
1555                 pipe->tc_time = port->time + params->tc_period;
1556         }
1557 }
1558
1559 #else
1560
1561 static inline uint32_t
1562 grinder_tc_ov_credits_update(struct rte_sched_port *port, uint32_t pos)
1563 {
1564         struct rte_sched_grinder *grinder = port->grinder + pos;
1565         struct rte_sched_subport *subport = grinder->subport;
1566         uint32_t tc_ov_consumption[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
1567         uint32_t tc_ov_consumption_max;
1568         uint32_t tc_ov_wm = subport->tc_ov_wm;
1569
1570         if (subport->tc_ov == 0)
1571                 return subport->tc_ov_wm_max;
1572
1573         tc_ov_consumption[0] = subport->tc_credits_per_period[0] - subport->tc_credits[0];
1574         tc_ov_consumption[1] = subport->tc_credits_per_period[1] - subport->tc_credits[1];
1575         tc_ov_consumption[2] = subport->tc_credits_per_period[2] - subport->tc_credits[2];
1576         tc_ov_consumption[3] = subport->tc_credits_per_period[3] - subport->tc_credits[3];
1577
1578         tc_ov_consumption_max = subport->tc_credits_per_period[3] -
1579                 (tc_ov_consumption[0] + tc_ov_consumption[1] + tc_ov_consumption[2]);
1580
1581         if (tc_ov_consumption[3] > (tc_ov_consumption_max - port->mtu)) {
1582                 tc_ov_wm  -= tc_ov_wm >> 7;
1583                 if (tc_ov_wm < subport->tc_ov_wm_min)
1584                         tc_ov_wm = subport->tc_ov_wm_min;
1585
1586                 return tc_ov_wm;
1587         }
1588
1589         tc_ov_wm += (tc_ov_wm >> 7) + 1;
1590         if (tc_ov_wm > subport->tc_ov_wm_max)
1591                 tc_ov_wm = subport->tc_ov_wm_max;
1592
1593         return tc_ov_wm;
1594 }
1595
1596 static inline void
1597 grinder_credits_update(struct rte_sched_port *port, uint32_t pos)
1598 {
1599         struct rte_sched_grinder *grinder = port->grinder + pos;
1600         struct rte_sched_subport *subport = grinder->subport;
1601         struct rte_sched_pipe *pipe = grinder->pipe;
1602         struct rte_sched_pipe_profile *params = grinder->pipe_params;
1603         uint64_t n_periods;
1604
1605         /* Subport TB */
1606         n_periods = (port->time - subport->tb_time) / subport->tb_period;
1607         subport->tb_credits += n_periods * subport->tb_credits_per_period;
1608         subport->tb_credits = rte_sched_min_val_2_u32(subport->tb_credits, subport->tb_size);
1609         subport->tb_time += n_periods * subport->tb_period;
1610
1611         /* Pipe TB */
1612         n_periods = (port->time - pipe->tb_time) / params->tb_period;
1613         pipe->tb_credits += n_periods * params->tb_credits_per_period;
1614         pipe->tb_credits = rte_sched_min_val_2_u32(pipe->tb_credits, params->tb_size);
1615         pipe->tb_time += n_periods * params->tb_period;
1616
1617         /* Subport TCs */
1618         if (unlikely(port->time >= subport->tc_time)) {
1619                 subport->tc_ov_wm = grinder_tc_ov_credits_update(port, pos);
1620
1621                 subport->tc_credits[0] = subport->tc_credits_per_period[0];
1622                 subport->tc_credits[1] = subport->tc_credits_per_period[1];
1623                 subport->tc_credits[2] = subport->tc_credits_per_period[2];
1624                 subport->tc_credits[3] = subport->tc_credits_per_period[3];
1625
1626                 subport->tc_time = port->time + subport->tc_period;
1627                 subport->tc_ov_period_id++;
1628         }
1629
1630         /* Pipe TCs */
1631         if (unlikely(port->time >= pipe->tc_time)) {
1632                 pipe->tc_credits[0] = params->tc_credits_per_period[0];
1633                 pipe->tc_credits[1] = params->tc_credits_per_period[1];
1634                 pipe->tc_credits[2] = params->tc_credits_per_period[2];
1635                 pipe->tc_credits[3] = params->tc_credits_per_period[3];
1636                 pipe->tc_time = port->time + params->tc_period;
1637         }
1638
1639         /* Pipe TCs - Oversubscription */
1640         if (unlikely(pipe->tc_ov_period_id != subport->tc_ov_period_id)) {
1641                 pipe->tc_ov_credits = subport->tc_ov_wm * params->tc_ov_weight;
1642
1643                 pipe->tc_ov_period_id = subport->tc_ov_period_id;
1644         }
1645 }
1646
1647 #endif /* RTE_SCHED_TS_CREDITS_UPDATE, RTE_SCHED_SUBPORT_TC_OV */
1648
1649
1650 #ifndef RTE_SCHED_SUBPORT_TC_OV
1651
1652 static inline int
1653 grinder_credits_check(struct rte_sched_port *port, uint32_t pos)
1654 {
1655         struct rte_sched_grinder *grinder = port->grinder + pos;
1656         struct rte_sched_subport *subport = grinder->subport;
1657         struct rte_sched_pipe *pipe = grinder->pipe;
1658         struct rte_mbuf *pkt = grinder->pkt;
1659         uint32_t tc_index = grinder->tc_index;
1660         uint32_t pkt_len = pkt->pkt_len + port->frame_overhead;
1661         uint32_t subport_tb_credits = subport->tb_credits;
1662         uint32_t subport_tc_credits = subport->tc_credits[tc_index];
1663         uint32_t pipe_tb_credits = pipe->tb_credits;
1664         uint32_t pipe_tc_credits = pipe->tc_credits[tc_index];
1665         int enough_credits;
1666
1667         /* Check queue credits */
1668         enough_credits = (pkt_len <= subport_tb_credits) &&
1669                 (pkt_len <= subport_tc_credits) &&
1670                 (pkt_len <= pipe_tb_credits) &&
1671                 (pkt_len <= pipe_tc_credits);
1672
1673         if (!enough_credits)
1674                 return 0;
1675
1676         /* Update port credits */
1677         subport->tb_credits -= pkt_len;
1678         subport->tc_credits[tc_index] -= pkt_len;
1679         pipe->tb_credits -= pkt_len;
1680         pipe->tc_credits[tc_index] -= pkt_len;
1681
1682         return 1;
1683 }
1684
1685 #else
1686
1687 static inline int
1688 grinder_credits_check(struct rte_sched_port *port, uint32_t pos)
1689 {
1690         struct rte_sched_grinder *grinder = port->grinder + pos;
1691         struct rte_sched_subport *subport = grinder->subport;
1692         struct rte_sched_pipe *pipe = grinder->pipe;
1693         struct rte_mbuf *pkt = grinder->pkt;
1694         uint32_t tc_index = grinder->tc_index;
1695         uint32_t pkt_len = pkt->pkt_len + port->frame_overhead;
1696         uint32_t subport_tb_credits = subport->tb_credits;
1697         uint32_t subport_tc_credits = subport->tc_credits[tc_index];
1698         uint32_t pipe_tb_credits = pipe->tb_credits;
1699         uint32_t pipe_tc_credits = pipe->tc_credits[tc_index];
1700         uint32_t pipe_tc_ov_mask1[] = {UINT32_MAX, UINT32_MAX, UINT32_MAX, pipe->tc_ov_credits};
1701         uint32_t pipe_tc_ov_mask2[] = {0, 0, 0, UINT32_MAX};
1702         uint32_t pipe_tc_ov_credits = pipe_tc_ov_mask1[tc_index];
1703         int enough_credits;
1704
1705         /* Check pipe and subport credits */
1706         enough_credits = (pkt_len <= subport_tb_credits) &&
1707                 (pkt_len <= subport_tc_credits) &&
1708                 (pkt_len <= pipe_tb_credits) &&
1709                 (pkt_len <= pipe_tc_credits) &&
1710                 (pkt_len <= pipe_tc_ov_credits);
1711
1712         if (!enough_credits)
1713                 return 0;
1714
1715         /* Update pipe and subport credits */
1716         subport->tb_credits -= pkt_len;
1717         subport->tc_credits[tc_index] -= pkt_len;
1718         pipe->tb_credits -= pkt_len;
1719         pipe->tc_credits[tc_index] -= pkt_len;
1720         pipe->tc_ov_credits -= pipe_tc_ov_mask2[tc_index] & pkt_len;
1721
1722         return 1;
1723 }
1724
1725 #endif /* RTE_SCHED_SUBPORT_TC_OV */
1726
1727
1728 static inline int
1729 grinder_schedule(struct rte_sched_port *port, uint32_t pos)
1730 {
1731         struct rte_sched_grinder *grinder = port->grinder + pos;
1732         struct rte_sched_queue *queue = grinder->queue[grinder->qpos];
1733         struct rte_mbuf *pkt = grinder->pkt;
1734         uint32_t pkt_len = pkt->pkt_len + port->frame_overhead;
1735
1736         if (!grinder_credits_check(port, pos))
1737                 return 0;
1738
1739         /* Advance port time */
1740         port->time += pkt_len;
1741
1742         /* Send packet */
1743         port->pkts_out[port->n_pkts_out++] = pkt;
1744         queue->qr++;
1745         grinder->wrr_tokens[grinder->qpos] += pkt_len * grinder->wrr_cost[grinder->qpos];
1746         if (queue->qr == queue->qw) {
1747                 uint32_t qindex = grinder->qindex[grinder->qpos];
1748
1749                 rte_bitmap_clear(port->bmp, qindex);
1750                 grinder->qmask &= ~(1 << grinder->qpos);
1751                 grinder->wrr_mask[grinder->qpos] = 0;
1752                 rte_sched_port_set_queue_empty_timestamp(port, qindex);
1753         }
1754
1755         /* Reset pipe loop detection */
1756         port->pipe_loop = RTE_SCHED_PIPE_INVALID;
1757         grinder->productive = 1;
1758
1759         return 1;
1760 }
1761
1762 #ifdef SCHED_VECTOR_SSE4
1763
1764 static inline int
1765 grinder_pipe_exists(struct rte_sched_port *port, uint32_t base_pipe)
1766 {
1767         __m128i index = _mm_set1_epi32(base_pipe);
1768         __m128i pipes = _mm_load_si128((__m128i *)port->grinder_base_bmp_pos);
1769         __m128i res = _mm_cmpeq_epi32(pipes, index);
1770
1771         pipes = _mm_load_si128((__m128i *)(port->grinder_base_bmp_pos + 4));
1772         pipes = _mm_cmpeq_epi32(pipes, index);
1773         res = _mm_or_si128(res, pipes);
1774
1775         if (_mm_testz_si128(res, res))
1776                 return 0;
1777
1778         return 1;
1779 }
1780
1781 #elif defined(SCHED_VECTOR_NEON)
1782
1783 static inline int
1784 grinder_pipe_exists(struct rte_sched_port *port, uint32_t base_pipe)
1785 {
1786         uint32x4_t index, pipes;
1787         uint32_t *pos = (uint32_t *)port->grinder_base_bmp_pos;
1788
1789         index = vmovq_n_u32(base_pipe);
1790         pipes = vld1q_u32(pos);
1791         if (!vminvq_u32(veorq_u32(pipes, index)))
1792                 return 1;
1793
1794         pipes = vld1q_u32(pos + 4);
1795         if (!vminvq_u32(veorq_u32(pipes, index)))
1796                 return 1;
1797
1798         return 0;
1799 }
1800
1801 #else
1802
1803 static inline int
1804 grinder_pipe_exists(struct rte_sched_port *port, uint32_t base_pipe)
1805 {
1806         uint32_t i;
1807
1808         for (i = 0; i < RTE_SCHED_PORT_N_GRINDERS; i++) {
1809                 if (port->grinder_base_bmp_pos[i] == base_pipe)
1810                         return 1;
1811         }
1812
1813         return 0;
1814 }
1815
1816 #endif /* RTE_SCHED_OPTIMIZATIONS */
1817
1818 static inline void
1819 grinder_pcache_populate(struct rte_sched_port *port, uint32_t pos, uint32_t bmp_pos, uint64_t bmp_slab)
1820 {
1821         struct rte_sched_grinder *grinder = port->grinder + pos;
1822         uint16_t w[4];
1823
1824         grinder->pcache_w = 0;
1825         grinder->pcache_r = 0;
1826
1827         w[0] = (uint16_t) bmp_slab;
1828         w[1] = (uint16_t) (bmp_slab >> 16);
1829         w[2] = (uint16_t) (bmp_slab >> 32);
1830         w[3] = (uint16_t) (bmp_slab >> 48);
1831
1832         grinder->pcache_qmask[grinder->pcache_w] = w[0];
1833         grinder->pcache_qindex[grinder->pcache_w] = bmp_pos;
1834         grinder->pcache_w += (w[0] != 0);
1835
1836         grinder->pcache_qmask[grinder->pcache_w] = w[1];
1837         grinder->pcache_qindex[grinder->pcache_w] = bmp_pos + 16;
1838         grinder->pcache_w += (w[1] != 0);
1839
1840         grinder->pcache_qmask[grinder->pcache_w] = w[2];
1841         grinder->pcache_qindex[grinder->pcache_w] = bmp_pos + 32;
1842         grinder->pcache_w += (w[2] != 0);
1843
1844         grinder->pcache_qmask[grinder->pcache_w] = w[3];
1845         grinder->pcache_qindex[grinder->pcache_w] = bmp_pos + 48;
1846         grinder->pcache_w += (w[3] != 0);
1847 }
1848
1849 static inline void
1850 grinder_tccache_populate(struct rte_sched_port *port, uint32_t pos, uint32_t qindex, uint16_t qmask)
1851 {
1852         struct rte_sched_grinder *grinder = port->grinder + pos;
1853         uint8_t b[4];
1854
1855         grinder->tccache_w = 0;
1856         grinder->tccache_r = 0;
1857
1858         b[0] = (uint8_t) (qmask & 0xF);
1859         b[1] = (uint8_t) ((qmask >> 4) & 0xF);
1860         b[2] = (uint8_t) ((qmask >> 8) & 0xF);
1861         b[3] = (uint8_t) ((qmask >> 12) & 0xF);
1862
1863         grinder->tccache_qmask[grinder->tccache_w] = b[0];
1864         grinder->tccache_qindex[grinder->tccache_w] = qindex;
1865         grinder->tccache_w += (b[0] != 0);
1866
1867         grinder->tccache_qmask[grinder->tccache_w] = b[1];
1868         grinder->tccache_qindex[grinder->tccache_w] = qindex + 4;
1869         grinder->tccache_w += (b[1] != 0);
1870
1871         grinder->tccache_qmask[grinder->tccache_w] = b[2];
1872         grinder->tccache_qindex[grinder->tccache_w] = qindex + 8;
1873         grinder->tccache_w += (b[2] != 0);
1874
1875         grinder->tccache_qmask[grinder->tccache_w] = b[3];
1876         grinder->tccache_qindex[grinder->tccache_w] = qindex + 12;
1877         grinder->tccache_w += (b[3] != 0);
1878 }
1879
1880 static inline int
1881 grinder_next_tc(struct rte_sched_port *port, uint32_t pos)
1882 {
1883         struct rte_sched_grinder *grinder = port->grinder + pos;
1884         struct rte_mbuf **qbase;
1885         uint32_t qindex;
1886         uint16_t qsize;
1887
1888         if (grinder->tccache_r == grinder->tccache_w)
1889                 return 0;
1890
1891         qindex = grinder->tccache_qindex[grinder->tccache_r];
1892         qbase = rte_sched_port_qbase(port, qindex);
1893         qsize = rte_sched_port_qsize(port, qindex);
1894
1895         grinder->tc_index = (qindex >> 2) & 0x3;
1896         grinder->qmask = grinder->tccache_qmask[grinder->tccache_r];
1897         grinder->qsize = qsize;
1898
1899         grinder->qindex[0] = qindex;
1900         grinder->qindex[1] = qindex + 1;
1901         grinder->qindex[2] = qindex + 2;
1902         grinder->qindex[3] = qindex + 3;
1903
1904         grinder->queue[0] = port->queue + qindex;
1905         grinder->queue[1] = port->queue + qindex + 1;
1906         grinder->queue[2] = port->queue + qindex + 2;
1907         grinder->queue[3] = port->queue + qindex + 3;
1908
1909         grinder->qbase[0] = qbase;
1910         grinder->qbase[1] = qbase + qsize;
1911         grinder->qbase[2] = qbase + 2 * qsize;
1912         grinder->qbase[3] = qbase + 3 * qsize;
1913
1914         grinder->tccache_r++;
1915         return 1;
1916 }
1917
1918 static inline int
1919 grinder_next_pipe(struct rte_sched_port *port, uint32_t pos)
1920 {
1921         struct rte_sched_grinder *grinder = port->grinder + pos;
1922         uint32_t pipe_qindex;
1923         uint16_t pipe_qmask;
1924
1925         if (grinder->pcache_r < grinder->pcache_w) {
1926                 pipe_qmask = grinder->pcache_qmask[grinder->pcache_r];
1927                 pipe_qindex = grinder->pcache_qindex[grinder->pcache_r];
1928                 grinder->pcache_r++;
1929         } else {
1930                 uint64_t bmp_slab = 0;
1931                 uint32_t bmp_pos = 0;
1932
1933                 /* Get another non-empty pipe group */
1934                 if (unlikely(rte_bitmap_scan(port->bmp, &bmp_pos, &bmp_slab) <= 0))
1935                         return 0;
1936
1937 #ifdef RTE_SCHED_DEBUG
1938                 debug_check_queue_slab(port, bmp_pos, bmp_slab);
1939 #endif
1940
1941                 /* Return if pipe group already in one of the other grinders */
1942                 port->grinder_base_bmp_pos[pos] = RTE_SCHED_BMP_POS_INVALID;
1943                 if (unlikely(grinder_pipe_exists(port, bmp_pos)))
1944                         return 0;
1945
1946                 port->grinder_base_bmp_pos[pos] = bmp_pos;
1947
1948                 /* Install new pipe group into grinder's pipe cache */
1949                 grinder_pcache_populate(port, pos, bmp_pos, bmp_slab);
1950
1951                 pipe_qmask = grinder->pcache_qmask[0];
1952                 pipe_qindex = grinder->pcache_qindex[0];
1953                 grinder->pcache_r = 1;
1954         }
1955
1956         /* Install new pipe in the grinder */
1957         grinder->pindex = pipe_qindex >> 4;
1958         grinder->subport = port->subport + (grinder->pindex / port->n_pipes_per_subport);
1959         grinder->pipe = port->pipe + grinder->pindex;
1960         grinder->pipe_params = NULL; /* to be set after the pipe structure is prefetched */
1961         grinder->productive = 0;
1962
1963         grinder_tccache_populate(port, pos, pipe_qindex, pipe_qmask);
1964         grinder_next_tc(port, pos);
1965
1966         /* Check for pipe exhaustion */
1967         if (grinder->pindex == port->pipe_loop) {
1968                 port->pipe_exhaustion = 1;
1969                 port->pipe_loop = RTE_SCHED_PIPE_INVALID;
1970         }
1971
1972         return 1;
1973 }
1974
1975
1976 static inline void
1977 grinder_wrr_load(struct rte_sched_port *port, uint32_t pos)
1978 {
1979         struct rte_sched_grinder *grinder = port->grinder + pos;
1980         struct rte_sched_pipe *pipe = grinder->pipe;
1981         struct rte_sched_pipe_profile *pipe_params = grinder->pipe_params;
1982         uint32_t tc_index = grinder->tc_index;
1983         uint32_t qmask = grinder->qmask;
1984         uint32_t qindex;
1985
1986         qindex = tc_index * 4;
1987
1988         grinder->wrr_tokens[0] = ((uint16_t) pipe->wrr_tokens[qindex]) << RTE_SCHED_WRR_SHIFT;
1989         grinder->wrr_tokens[1] = ((uint16_t) pipe->wrr_tokens[qindex + 1]) << RTE_SCHED_WRR_SHIFT;
1990         grinder->wrr_tokens[2] = ((uint16_t) pipe->wrr_tokens[qindex + 2]) << RTE_SCHED_WRR_SHIFT;
1991         grinder->wrr_tokens[3] = ((uint16_t) pipe->wrr_tokens[qindex + 3]) << RTE_SCHED_WRR_SHIFT;
1992
1993         grinder->wrr_mask[0] = (qmask & 0x1) * 0xFFFF;
1994         grinder->wrr_mask[1] = ((qmask >> 1) & 0x1) * 0xFFFF;
1995         grinder->wrr_mask[2] = ((qmask >> 2) & 0x1) * 0xFFFF;
1996         grinder->wrr_mask[3] = ((qmask >> 3) & 0x1) * 0xFFFF;
1997
1998         grinder->wrr_cost[0] = pipe_params->wrr_cost[qindex];
1999         grinder->wrr_cost[1] = pipe_params->wrr_cost[qindex + 1];
2000         grinder->wrr_cost[2] = pipe_params->wrr_cost[qindex + 2];
2001         grinder->wrr_cost[3] = pipe_params->wrr_cost[qindex + 3];
2002 }
2003
2004 static inline void
2005 grinder_wrr_store(struct rte_sched_port *port, uint32_t pos)
2006 {
2007         struct rte_sched_grinder *grinder = port->grinder + pos;
2008         struct rte_sched_pipe *pipe = grinder->pipe;
2009         uint32_t tc_index = grinder->tc_index;
2010         uint32_t qindex;
2011
2012         qindex = tc_index * 4;
2013
2014         pipe->wrr_tokens[qindex] = (grinder->wrr_tokens[0] & grinder->wrr_mask[0])
2015                 >> RTE_SCHED_WRR_SHIFT;
2016         pipe->wrr_tokens[qindex + 1] = (grinder->wrr_tokens[1] & grinder->wrr_mask[1])
2017                 >> RTE_SCHED_WRR_SHIFT;
2018         pipe->wrr_tokens[qindex + 2] = (grinder->wrr_tokens[2] & grinder->wrr_mask[2])
2019                 >> RTE_SCHED_WRR_SHIFT;
2020         pipe->wrr_tokens[qindex + 3] = (grinder->wrr_tokens[3] & grinder->wrr_mask[3])
2021                 >> RTE_SCHED_WRR_SHIFT;
2022 }
2023
2024 static inline void
2025 grinder_wrr(struct rte_sched_port *port, uint32_t pos)
2026 {
2027         struct rte_sched_grinder *grinder = port->grinder + pos;
2028         uint16_t wrr_tokens_min;
2029
2030         grinder->wrr_tokens[0] |= ~grinder->wrr_mask[0];
2031         grinder->wrr_tokens[1] |= ~grinder->wrr_mask[1];
2032         grinder->wrr_tokens[2] |= ~grinder->wrr_mask[2];
2033         grinder->wrr_tokens[3] |= ~grinder->wrr_mask[3];
2034
2035         grinder->qpos = rte_min_pos_4_u16(grinder->wrr_tokens);
2036         wrr_tokens_min = grinder->wrr_tokens[grinder->qpos];
2037
2038         grinder->wrr_tokens[0] -= wrr_tokens_min;
2039         grinder->wrr_tokens[1] -= wrr_tokens_min;
2040         grinder->wrr_tokens[2] -= wrr_tokens_min;
2041         grinder->wrr_tokens[3] -= wrr_tokens_min;
2042 }
2043
2044
2045 #define grinder_evict(port, pos)
2046
2047 static inline void
2048 grinder_prefetch_pipe(struct rte_sched_port *port, uint32_t pos)
2049 {
2050         struct rte_sched_grinder *grinder = port->grinder + pos;
2051
2052         rte_prefetch0(grinder->pipe);
2053         rte_prefetch0(grinder->queue[0]);
2054 }
2055
2056 static inline void
2057 grinder_prefetch_tc_queue_arrays(struct rte_sched_port *port, uint32_t pos)
2058 {
2059         struct rte_sched_grinder *grinder = port->grinder + pos;
2060         uint16_t qsize, qr[4];
2061
2062         qsize = grinder->qsize;
2063         qr[0] = grinder->queue[0]->qr & (qsize - 1);
2064         qr[1] = grinder->queue[1]->qr & (qsize - 1);
2065         qr[2] = grinder->queue[2]->qr & (qsize - 1);
2066         qr[3] = grinder->queue[3]->qr & (qsize - 1);
2067
2068         rte_prefetch0(grinder->qbase[0] + qr[0]);
2069         rte_prefetch0(grinder->qbase[1] + qr[1]);
2070
2071         grinder_wrr_load(port, pos);
2072         grinder_wrr(port, pos);
2073
2074         rte_prefetch0(grinder->qbase[2] + qr[2]);
2075         rte_prefetch0(grinder->qbase[3] + qr[3]);
2076 }
2077
2078 static inline void
2079 grinder_prefetch_mbuf(struct rte_sched_port *port, uint32_t pos)
2080 {
2081         struct rte_sched_grinder *grinder = port->grinder + pos;
2082         uint32_t qpos = grinder->qpos;
2083         struct rte_mbuf **qbase = grinder->qbase[qpos];
2084         uint16_t qsize = grinder->qsize;
2085         uint16_t qr = grinder->queue[qpos]->qr & (qsize - 1);
2086
2087         grinder->pkt = qbase[qr];
2088         rte_prefetch0(grinder->pkt);
2089
2090         if (unlikely((qr & 0x7) == 7)) {
2091                 uint16_t qr_next = (grinder->queue[qpos]->qr + 1) & (qsize - 1);
2092
2093                 rte_prefetch0(qbase + qr_next);
2094         }
2095 }
2096
2097 static inline uint32_t
2098 grinder_handle(struct rte_sched_port *port, uint32_t pos)
2099 {
2100         struct rte_sched_grinder *grinder = port->grinder + pos;
2101
2102         switch (grinder->state) {
2103         case e_GRINDER_PREFETCH_PIPE:
2104         {
2105                 if (grinder_next_pipe(port, pos)) {
2106                         grinder_prefetch_pipe(port, pos);
2107                         port->busy_grinders++;
2108
2109                         grinder->state = e_GRINDER_PREFETCH_TC_QUEUE_ARRAYS;
2110                         return 0;
2111                 }
2112
2113                 return 0;
2114         }
2115
2116         case e_GRINDER_PREFETCH_TC_QUEUE_ARRAYS:
2117         {
2118                 struct rte_sched_pipe *pipe = grinder->pipe;
2119
2120                 grinder->pipe_params = port->pipe_profiles + pipe->profile;
2121                 grinder_prefetch_tc_queue_arrays(port, pos);
2122                 grinder_credits_update(port, pos);
2123
2124                 grinder->state = e_GRINDER_PREFETCH_MBUF;
2125                 return 0;
2126         }
2127
2128         case e_GRINDER_PREFETCH_MBUF:
2129         {
2130                 grinder_prefetch_mbuf(port, pos);
2131
2132                 grinder->state = e_GRINDER_READ_MBUF;
2133                 return 0;
2134         }
2135
2136         case e_GRINDER_READ_MBUF:
2137         {
2138                 uint32_t result = 0;
2139
2140                 result = grinder_schedule(port, pos);
2141
2142                 /* Look for next packet within the same TC */
2143                 if (result && grinder->qmask) {
2144                         grinder_wrr(port, pos);
2145                         grinder_prefetch_mbuf(port, pos);
2146
2147                         return 1;
2148                 }
2149                 grinder_wrr_store(port, pos);
2150
2151                 /* Look for another active TC within same pipe */
2152                 if (grinder_next_tc(port, pos)) {
2153                         grinder_prefetch_tc_queue_arrays(port, pos);
2154
2155                         grinder->state = e_GRINDER_PREFETCH_MBUF;
2156                         return result;
2157                 }
2158
2159                 if (grinder->productive == 0 &&
2160                     port->pipe_loop == RTE_SCHED_PIPE_INVALID)
2161                         port->pipe_loop = grinder->pindex;
2162
2163                 grinder_evict(port, pos);
2164
2165                 /* Look for another active pipe */
2166                 if (grinder_next_pipe(port, pos)) {
2167                         grinder_prefetch_pipe(port, pos);
2168
2169                         grinder->state = e_GRINDER_PREFETCH_TC_QUEUE_ARRAYS;
2170                         return result;
2171                 }
2172
2173                 /* No active pipe found */
2174                 port->busy_grinders--;
2175
2176                 grinder->state = e_GRINDER_PREFETCH_PIPE;
2177                 return result;
2178         }
2179
2180         default:
2181                 rte_panic("Algorithmic error (invalid state)\n");
2182                 return 0;
2183         }
2184 }
2185
2186 static inline void
2187 rte_sched_port_time_resync(struct rte_sched_port *port)
2188 {
2189         uint64_t cycles = rte_get_tsc_cycles();
2190         uint64_t cycles_diff = cycles - port->time_cpu_cycles;
2191         uint64_t bytes_diff;
2192
2193         /* Compute elapsed time in bytes */
2194         bytes_diff = rte_reciprocal_divide(cycles_diff << RTE_SCHED_TIME_SHIFT,
2195                                            port->inv_cycles_per_byte);
2196
2197         /* Advance port time */
2198         port->time_cpu_cycles = cycles;
2199         port->time_cpu_bytes += bytes_diff;
2200         if (port->time < port->time_cpu_bytes)
2201                 port->time = port->time_cpu_bytes;
2202
2203         /* Reset pipe loop detection */
2204         port->pipe_loop = RTE_SCHED_PIPE_INVALID;
2205 }
2206
2207 static inline int
2208 rte_sched_port_exceptions(struct rte_sched_port *port, int second_pass)
2209 {
2210         int exceptions;
2211
2212         /* Check if any exception flag is set */
2213         exceptions = (second_pass && port->busy_grinders == 0) ||
2214                 (port->pipe_exhaustion == 1);
2215
2216         /* Clear exception flags */
2217         port->pipe_exhaustion = 0;
2218
2219         return exceptions;
2220 }
2221
2222 int
2223 rte_sched_port_dequeue(struct rte_sched_port *port, struct rte_mbuf **pkts, uint32_t n_pkts)
2224 {
2225         uint32_t i, count;
2226
2227         port->pkts_out = pkts;
2228         port->n_pkts_out = 0;
2229
2230         rte_sched_port_time_resync(port);
2231
2232         /* Take each queue in the grinder one step further */
2233         for (i = 0, count = 0; ; i++)  {
2234                 count += grinder_handle(port, i & (RTE_SCHED_PORT_N_GRINDERS - 1));
2235                 if ((count == n_pkts) ||
2236                     rte_sched_port_exceptions(port, i >= RTE_SCHED_PORT_N_GRINDERS)) {
2237                         break;
2238                 }
2239         }
2240
2241         return count;
2242 }