2 * Copyright(c) 2016 Intel Corporation. All rights reserved.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
19 #include <sys/mount.h>
23 #include <vppinfra/vec.h>
24 #include <vppinfra/error.h>
25 #include <vppinfra/format.h>
26 #include <vppinfra/bitmap.h>
28 #include <vnet/vnet.h>
29 #include <vnet/ethernet/ethernet.h>
30 #include <dpdk/device/dpdk.h>
32 #include <vlib/unix/physmem.h>
33 #include <vlib/pci/pci.h>
34 #include <vlibmemory/api.h>
35 #include <vlibmemory/vl_memory_msg_enum.h> /* enumerate all vlib messages */
37 #define vl_typedefs /* define message structures */
38 #include <vlibmemory/vl_memory_api_h.h>
41 /* instantiate all the print functions we know about */
42 #define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__)
44 #include <vlibmemory/vl_memory_api_h.h>
47 #include <dpdk/device/dpdk_priv.h>
49 dpdk_main_t dpdk_main;
53 * HQoS default configuration values
57 static dpdk_device_config_hqos_t hqos_params_default = {
58 .hqos_thread_valid = 0,
65 * Packet field to identify the subport.
67 * Default value: Since only one subport is defined by default (see below:
68 * n_subports_per_port = 1), the subport ID is hardcoded to 0.
70 .pktfield0_slabpos = 0,
71 .pktfield0_slabmask = 0,
74 * Packet field to identify the pipe.
76 * Default value: Assuming Ethernet/IPv4/UDP packets, UDP payload bits 12 .. 23
78 .pktfield1_slabpos = 40,
79 .pktfield1_slabmask = 0x0000000FFF000000LLU,
81 /* Packet field used as index into TC translation table to identify the traffic
84 * Default value: Assuming Ethernet/IPv4 packets, IPv4 DSCP field
86 .pktfield2_slabpos = 8,
87 .pktfield2_slabmask = 0x00000000000000FCLLU,
89 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
90 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
91 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
92 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
97 .name = NULL, /* Set at init */
98 .socket = 0, /* Set at init */
99 .rate = 1250000000, /* Assuming 10GbE port */
100 .mtu = 14 + 1500, /* Assuming Ethernet/IPv4 pkt (Ethernet FCS not included) */
101 .frame_overhead = RTE_SCHED_FRAME_OVERHEAD_DEFAULT,
102 .n_subports_per_port = 1,
103 .n_pipes_per_subport = 4096,
104 .qsize = {64, 64, 64, 64},
105 .pipe_profiles = NULL, /* Set at config */
106 .n_pipe_profiles = 1,
110 /* Traffic Class 0 Colors Green / Yellow / Red */
111 [0][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
113 [0][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
115 [0][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
118 /* Traffic Class 1 - Colors Green / Yellow / Red */
119 [1][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
121 [1][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
123 [1][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
126 /* Traffic Class 2 - Colors Green / Yellow / Red */
127 [2][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
129 [2][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
131 [2][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
134 /* Traffic Class 3 - Colors Green / Yellow / Red */
135 [3][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
137 [3][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
139 [3][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
142 #endif /* RTE_SCHED_RED */
146 static struct rte_sched_subport_params hqos_subport_params_default = {
147 .tb_rate = 1250000000, /* 10GbE line rate (measured in bytes/second) */
149 .tc_rate = {1250000000, 1250000000, 1250000000, 1250000000},
153 static struct rte_sched_pipe_params hqos_pipe_params_default = {
154 .tb_rate = 305175, /* 10GbE line rate divided by 4K pipes */
156 .tc_rate = {305175, 305175, 305175, 305175},
158 #ifdef RTE_SCHED_SUBPORT_TC_OV
161 .wrr_weights = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
171 dpdk_hqos_validate_mask (u64 mask, u32 n)
173 int count = __builtin_popcountll (mask);
174 int pos_lead = sizeof (u64) * 8 - __builtin_clzll (mask);
175 int pos_trail = __builtin_ctzll (mask);
176 int count_expected = __builtin_popcount (n - 1);
178 /* Handle the exceptions */
180 return -1; /* Error */
182 if ((mask == 0) && (n == 1))
185 if (((mask == 0) && (n != 1)) || ((mask != 0) && (n == 1)))
186 return -2; /* Error */
188 /* Check that mask is contiguous */
189 if ((pos_lead - pos_trail) != count)
190 return -3; /* Error */
192 /* Check that mask contains the expected number of bits set */
193 if (count != count_expected)
194 return -4; /* Error */
200 dpdk_device_config_hqos_pipe_profile_default (dpdk_device_config_hqos_t *
201 hqos, u32 pipe_profile_id)
203 memcpy (&hqos->pipe[pipe_profile_id], &hqos_pipe_params_default,
204 sizeof (hqos_pipe_params_default));
208 dpdk_device_config_hqos_default (dpdk_device_config_hqos_t * hqos)
210 struct rte_sched_subport_params *subport_params;
211 struct rte_sched_pipe_params *pipe_params;
215 memcpy (hqos, &hqos_params_default, sizeof (hqos_params_default));
218 vec_add2 (hqos->pipe, pipe_params, hqos->port.n_pipe_profiles);
220 for (i = 0; i < vec_len (hqos->pipe); i++)
221 memcpy (&pipe_params[i],
222 &hqos_pipe_params_default, sizeof (hqos_pipe_params_default));
224 hqos->port.pipe_profiles = hqos->pipe;
227 vec_add2 (hqos->subport, subport_params, hqos->port.n_subports_per_port);
229 for (i = 0; i < vec_len (hqos->subport); i++)
230 memcpy (&subport_params[i],
231 &hqos_subport_params_default,
232 sizeof (hqos_subport_params_default));
235 vec_add2 (hqos->pipe_map,
237 hqos->port.n_subports_per_port * hqos->port.n_pipes_per_subport);
239 for (i = 0; i < vec_len (hqos->pipe_map); i++)
250 dpdk_port_setup_hqos (dpdk_device_t * xd, dpdk_device_config_hqos_t * hqos)
252 vlib_thread_main_t *tm = vlib_get_thread_main ();
257 /* Detect the set of worker threads */
258 int worker_thread_first = 0;
259 int worker_thread_count = 0;
261 uword *p = hash_get_mem (tm->thread_registrations_by_name, "workers");
262 vlib_thread_registration_t *tr =
263 p ? (vlib_thread_registration_t *) p[0] : 0;
265 if (tr && tr->count > 0)
267 worker_thread_first = tr->first_index;
268 worker_thread_count = tr->count;
271 /* Allocate the per-thread device data array */
272 vec_validate_aligned (xd->hqos_wt, tm->n_vlib_mains - 1,
273 CLIB_CACHE_LINE_BYTES);
274 memset (xd->hqos_wt, 0, tm->n_vlib_mains * sizeof (xd->hqos_wt[0]));
276 vec_validate_aligned (xd->hqos_ht, 0, CLIB_CACHE_LINE_BYTES);
277 memset (xd->hqos_ht, 0, sizeof (xd->hqos_ht[0]));
279 /* Allocate space for one SWQ per worker thread in the I/O TX thread data structure */
280 vec_validate (xd->hqos_ht->swq, worker_thread_count);
283 for (i = 0; i < worker_thread_count + 1; i++)
285 u32 swq_flags = RING_F_SP_ENQ | RING_F_SC_DEQ;
287 snprintf (name, sizeof (name), "SWQ-worker%u-to-device%u", i,
289 xd->hqos_ht->swq[i] =
290 rte_ring_create (name, hqos->swq_size, xd->cpu_socket, swq_flags);
291 if (xd->hqos_ht->swq[i] == NULL)
292 return clib_error_return (0,
293 "SWQ-worker%u-to-device%u: rte_ring_create err",
294 i, xd->device_index);
302 snprintf (name, sizeof (name), "HQoS%u", xd->device_index);
303 hqos->port.name = strdup (name);
304 if (hqos->port.name == NULL)
305 return clib_error_return (0, "HQoS%u: strdup err", xd->device_index);
307 hqos->port.socket = rte_eth_dev_socket_id (xd->device_index);
308 if (hqos->port.socket == SOCKET_ID_ANY)
309 hqos->port.socket = 0;
311 xd->hqos_ht->hqos = rte_sched_port_config (&hqos->port);
312 if (xd->hqos_ht->hqos == NULL)
313 return clib_error_return (0, "HQoS%u: rte_sched_port_config err",
317 for (subport_id = 0; subport_id < hqos->port.n_subports_per_port;
323 rte_sched_subport_config (xd->hqos_ht->hqos, subport_id,
324 &hqos->subport[subport_id]);
326 return clib_error_return (0,
327 "HQoS%u subport %u: rte_sched_subport_config err (%d)",
328 xd->device_index, subport_id, rv);
331 for (pipe_id = 0; pipe_id < hqos->port.n_pipes_per_subport; pipe_id++)
333 u32 pos = subport_id * hqos->port.n_pipes_per_subport + pipe_id;
334 u32 profile_id = hqos->pipe_map[pos];
337 rte_sched_pipe_config (xd->hqos_ht->hqos, subport_id, pipe_id,
340 return clib_error_return (0,
341 "HQoS%u subport %u pipe %u: rte_sched_pipe_config err (%d)",
342 xd->device_index, subport_id, pipe_id,
347 /* Set up per-thread device data for the I/O TX thread */
348 xd->hqos_ht->hqos_burst_enq = hqos->burst_enq;
349 xd->hqos_ht->hqos_burst_deq = hqos->burst_deq;
350 vec_validate (xd->hqos_ht->pkts_enq, 2 * hqos->burst_enq - 1);
351 vec_validate (xd->hqos_ht->pkts_deq, hqos->burst_deq - 1);
352 xd->hqos_ht->pkts_enq_len = 0;
353 xd->hqos_ht->swq_pos = 0;
354 xd->hqos_ht->flush_count = 0;
356 /* Set up per-thread device data for each worker thread */
357 for (i = 0; i < worker_thread_count + 1; i++)
361 tid = worker_thread_first + (i - 1);
365 xd->hqos_wt[tid].swq = xd->hqos_ht->swq[i];
366 xd->hqos_wt[tid].hqos_field0_slabpos = hqos->pktfield0_slabpos;
367 xd->hqos_wt[tid].hqos_field0_slabmask = hqos->pktfield0_slabmask;
368 xd->hqos_wt[tid].hqos_field0_slabshr =
369 __builtin_ctzll (hqos->pktfield0_slabmask);
370 xd->hqos_wt[tid].hqos_field1_slabpos = hqos->pktfield1_slabpos;
371 xd->hqos_wt[tid].hqos_field1_slabmask = hqos->pktfield1_slabmask;
372 xd->hqos_wt[tid].hqos_field1_slabshr =
373 __builtin_ctzll (hqos->pktfield1_slabmask);
374 xd->hqos_wt[tid].hqos_field2_slabpos = hqos->pktfield2_slabpos;
375 xd->hqos_wt[tid].hqos_field2_slabmask = hqos->pktfield2_slabmask;
376 xd->hqos_wt[tid].hqos_field2_slabshr =
377 __builtin_ctzll (hqos->pktfield2_slabmask);
378 memcpy (xd->hqos_wt[tid].hqos_tc_table, hqos->tc_table,
379 sizeof (hqos->tc_table));
391 * dpdk_hqos_thread - Contains the main loop of an HQoS thread.
394 * Information for the current thread
396 static_always_inline void
397 dpdk_hqos_thread_internal_hqos_dbg_bypass (vlib_main_t * vm)
399 dpdk_main_t *dm = &dpdk_main;
400 u32 thread_index = vm->thread_index;
406 vlib_worker_thread_barrier_check ();
408 u32 n_devs = vec_len (dm->devices_by_hqos_cpu[thread_index]);
409 if (dev_pos >= n_devs)
412 dpdk_device_and_queue_t *dq =
413 vec_elt_at_index (dm->devices_by_hqos_cpu[thread_index], dev_pos);
414 dpdk_device_t *xd = vec_elt_at_index (dm->devices, dq->device);
416 dpdk_device_hqos_per_hqos_thread_t *hqos = xd->hqos_ht;
417 u32 device_index = xd->device_index;
418 u16 queue_id = dq->queue_id;
420 struct rte_mbuf **pkts_enq = hqos->pkts_enq;
421 u32 pkts_enq_len = hqos->pkts_enq_len;
422 u32 swq_pos = hqos->swq_pos;
423 u32 n_swq = vec_len (hqos->swq), i;
424 u32 flush_count = hqos->flush_count;
426 for (i = 0; i < n_swq; i++)
428 /* Get current SWQ for this device */
429 struct rte_ring *swq = hqos->swq[swq_pos];
431 /* Read SWQ burst to packet buffer of this device */
432 pkts_enq_len += rte_ring_sc_dequeue_burst (swq,
434 &pkts_enq[pkts_enq_len],
435 hqos->hqos_burst_enq);
437 /* Get next SWQ for this device */
439 if (swq_pos >= n_swq)
441 hqos->swq_pos = swq_pos;
443 /* HWQ TX enqueue when burst available */
444 if (pkts_enq_len >= hqos->hqos_burst_enq)
446 u32 n_pkts = rte_eth_tx_burst (device_index,
449 (uint16_t) pkts_enq_len);
451 for (; n_pkts < pkts_enq_len; n_pkts++)
452 rte_pktmbuf_free (pkts_enq[n_pkts]);
462 if (PREDICT_FALSE (flush_count == HQOS_FLUSH_COUNT_THRESHOLD))
464 rte_sched_port_enqueue (hqos->hqos, pkts_enq, pkts_enq_len);
470 hqos->pkts_enq_len = pkts_enq_len;
471 hqos->flush_count = flush_count;
473 /* Advance to next device */
478 static_always_inline void
479 dpdk_hqos_thread_internal (vlib_main_t * vm)
481 dpdk_main_t *dm = &dpdk_main;
482 u32 thread_index = vm->thread_index;
488 vlib_worker_thread_barrier_check ();
490 u32 n_devs = vec_len (dm->devices_by_hqos_cpu[thread_index]);
491 if (PREDICT_FALSE (n_devs == 0))
496 if (dev_pos >= n_devs)
499 dpdk_device_and_queue_t *dq =
500 vec_elt_at_index (dm->devices_by_hqos_cpu[thread_index], dev_pos);
501 dpdk_device_t *xd = vec_elt_at_index (dm->devices, dq->device);
503 dpdk_device_hqos_per_hqos_thread_t *hqos = xd->hqos_ht;
504 u32 device_index = xd->device_index;
505 u16 queue_id = dq->queue_id;
507 struct rte_mbuf **pkts_enq = hqos->pkts_enq;
508 struct rte_mbuf **pkts_deq = hqos->pkts_deq;
509 u32 pkts_enq_len = hqos->pkts_enq_len;
510 u32 swq_pos = hqos->swq_pos;
511 u32 n_swq = vec_len (hqos->swq), i;
512 u32 flush_count = hqos->flush_count;
515 * SWQ dequeue and HQoS enqueue for current device
517 for (i = 0; i < n_swq; i++)
519 /* Get current SWQ for this device */
520 struct rte_ring *swq = hqos->swq[swq_pos];
522 /* Read SWQ burst to packet buffer of this device */
523 pkts_enq_len += rte_ring_sc_dequeue_burst (swq,
525 &pkts_enq[pkts_enq_len],
526 hqos->hqos_burst_enq);
528 /* Get next SWQ for this device */
530 if (swq_pos >= n_swq)
532 hqos->swq_pos = swq_pos;
534 /* HQoS enqueue when burst available */
535 if (pkts_enq_len >= hqos->hqos_burst_enq)
537 rte_sched_port_enqueue (hqos->hqos, pkts_enq, pkts_enq_len);
547 if (PREDICT_FALSE (flush_count == HQOS_FLUSH_COUNT_THRESHOLD))
549 rte_sched_port_enqueue (hqos->hqos, pkts_enq, pkts_enq_len);
555 hqos->pkts_enq_len = pkts_enq_len;
556 hqos->flush_count = flush_count;
559 * HQoS dequeue and HWQ TX enqueue for current device
562 u32 pkts_deq_len, n_pkts;
564 pkts_deq_len = rte_sched_port_dequeue (hqos->hqos,
566 hqos->hqos_burst_deq);
568 for (n_pkts = 0; n_pkts < pkts_deq_len;)
569 n_pkts += rte_eth_tx_burst (device_index,
572 (uint16_t) (pkts_deq_len - n_pkts));
575 /* Advance to next device */
581 dpdk_hqos_thread (vlib_worker_thread_t * w)
584 vlib_thread_main_t *tm = vlib_get_thread_main ();
585 dpdk_main_t *dm = &dpdk_main;
587 vm = vlib_get_main ();
589 ASSERT (vm->thread_index == vlib_get_thread_index ());
591 clib_time_init (&vm->clib_time);
592 clib_mem_set_heap (w->thread_mheap);
594 /* Wait until the dpdk init sequence is complete */
595 while (tm->worker_thread_release == 0)
596 vlib_worker_thread_barrier_check ();
598 if (vec_len (dm->devices_by_hqos_cpu[vm->thread_index]) == 0)
601 ("current I/O TX thread does not have any devices assigned to it");
603 if (DPDK_HQOS_DBG_BYPASS)
604 dpdk_hqos_thread_internal_hqos_dbg_bypass (vm);
606 dpdk_hqos_thread_internal (vm);
610 dpdk_hqos_thread_fn (void *arg)
612 vlib_worker_thread_t *w = (vlib_worker_thread_t *) arg;
613 vlib_worker_thread_init (w);
614 dpdk_hqos_thread (w);
618 VLIB_REGISTER_THREAD (hqos_thread_reg, static) =
620 .name = "hqos-threads",
621 .short_name = "hqos-threads",
622 .function = dpdk_hqos_thread_fn,
627 * HQoS run-time code to be called by the worker threads
629 #define BITFIELD(byte_array, slab_pos, slab_mask, slab_shr) \
631 u64 slab = *((u64 *) &byte_array[slab_pos]); \
632 u64 val = (rte_be_to_cpu_64(slab) & slab_mask) >> slab_shr; \
636 #define RTE_SCHED_PORT_HIERARCHY(subport, pipe, traffic_class, queue, color) \
637 ((((u64) (queue)) & 0x3) | \
638 ((((u64) (traffic_class)) & 0x3) << 2) | \
639 ((((u64) (color)) & 0x3) << 4) | \
640 ((((u64) (subport)) & 0xFFFF) << 16) | \
641 ((((u64) (pipe)) & 0xFFFFFFFF) << 32))
644 dpdk_hqos_metadata_set (dpdk_device_hqos_per_worker_thread_t * hqos,
645 struct rte_mbuf **pkts, u32 n_pkts)
649 for (i = 0; i < (n_pkts & (~0x3)); i += 4)
651 struct rte_mbuf *pkt0 = pkts[i];
652 struct rte_mbuf *pkt1 = pkts[i + 1];
653 struct rte_mbuf *pkt2 = pkts[i + 2];
654 struct rte_mbuf *pkt3 = pkts[i + 3];
656 u8 *pkt0_data = rte_pktmbuf_mtod (pkt0, u8 *);
657 u8 *pkt1_data = rte_pktmbuf_mtod (pkt1, u8 *);
658 u8 *pkt2_data = rte_pktmbuf_mtod (pkt2, u8 *);
659 u8 *pkt3_data = rte_pktmbuf_mtod (pkt3, u8 *);
661 u64 pkt0_subport = BITFIELD (pkt0_data, hqos->hqos_field0_slabpos,
662 hqos->hqos_field0_slabmask,
663 hqos->hqos_field0_slabshr);
664 u64 pkt0_pipe = BITFIELD (pkt0_data, hqos->hqos_field1_slabpos,
665 hqos->hqos_field1_slabmask,
666 hqos->hqos_field1_slabshr);
667 u64 pkt0_dscp = BITFIELD (pkt0_data, hqos->hqos_field2_slabpos,
668 hqos->hqos_field2_slabmask,
669 hqos->hqos_field2_slabshr);
670 u32 pkt0_tc = hqos->hqos_tc_table[pkt0_dscp & 0x3F] >> 2;
671 u32 pkt0_tc_q = hqos->hqos_tc_table[pkt0_dscp & 0x3F] & 0x3;
673 u64 pkt1_subport = BITFIELD (pkt1_data, hqos->hqos_field0_slabpos,
674 hqos->hqos_field0_slabmask,
675 hqos->hqos_field0_slabshr);
676 u64 pkt1_pipe = BITFIELD (pkt1_data, hqos->hqos_field1_slabpos,
677 hqos->hqos_field1_slabmask,
678 hqos->hqos_field1_slabshr);
679 u64 pkt1_dscp = BITFIELD (pkt1_data, hqos->hqos_field2_slabpos,
680 hqos->hqos_field2_slabmask,
681 hqos->hqos_field2_slabshr);
682 u32 pkt1_tc = hqos->hqos_tc_table[pkt1_dscp & 0x3F] >> 2;
683 u32 pkt1_tc_q = hqos->hqos_tc_table[pkt1_dscp & 0x3F] & 0x3;
685 u64 pkt2_subport = BITFIELD (pkt2_data, hqos->hqos_field0_slabpos,
686 hqos->hqos_field0_slabmask,
687 hqos->hqos_field0_slabshr);
688 u64 pkt2_pipe = BITFIELD (pkt2_data, hqos->hqos_field1_slabpos,
689 hqos->hqos_field1_slabmask,
690 hqos->hqos_field1_slabshr);
691 u64 pkt2_dscp = BITFIELD (pkt2_data, hqos->hqos_field2_slabpos,
692 hqos->hqos_field2_slabmask,
693 hqos->hqos_field2_slabshr);
694 u32 pkt2_tc = hqos->hqos_tc_table[pkt2_dscp & 0x3F] >> 2;
695 u32 pkt2_tc_q = hqos->hqos_tc_table[pkt2_dscp & 0x3F] & 0x3;
697 u64 pkt3_subport = BITFIELD (pkt3_data, hqos->hqos_field0_slabpos,
698 hqos->hqos_field0_slabmask,
699 hqos->hqos_field0_slabshr);
700 u64 pkt3_pipe = BITFIELD (pkt3_data, hqos->hqos_field1_slabpos,
701 hqos->hqos_field1_slabmask,
702 hqos->hqos_field1_slabshr);
703 u64 pkt3_dscp = BITFIELD (pkt3_data, hqos->hqos_field2_slabpos,
704 hqos->hqos_field2_slabmask,
705 hqos->hqos_field2_slabshr);
706 u32 pkt3_tc = hqos->hqos_tc_table[pkt3_dscp & 0x3F] >> 2;
707 u32 pkt3_tc_q = hqos->hqos_tc_table[pkt3_dscp & 0x3F] & 0x3;
709 u64 pkt0_sched = RTE_SCHED_PORT_HIERARCHY (pkt0_subport,
714 u64 pkt1_sched = RTE_SCHED_PORT_HIERARCHY (pkt1_subport,
719 u64 pkt2_sched = RTE_SCHED_PORT_HIERARCHY (pkt2_subport,
724 u64 pkt3_sched = RTE_SCHED_PORT_HIERARCHY (pkt3_subport,
730 pkt0->hash.sched.lo = pkt0_sched & 0xFFFFFFFF;
731 pkt0->hash.sched.hi = pkt0_sched >> 32;
732 pkt1->hash.sched.lo = pkt1_sched & 0xFFFFFFFF;
733 pkt1->hash.sched.hi = pkt1_sched >> 32;
734 pkt2->hash.sched.lo = pkt2_sched & 0xFFFFFFFF;
735 pkt2->hash.sched.hi = pkt2_sched >> 32;
736 pkt3->hash.sched.lo = pkt3_sched & 0xFFFFFFFF;
737 pkt3->hash.sched.hi = pkt3_sched >> 32;
740 for (; i < n_pkts; i++)
742 struct rte_mbuf *pkt = pkts[i];
744 u8 *pkt_data = rte_pktmbuf_mtod (pkt, u8 *);
746 u64 pkt_subport = BITFIELD (pkt_data, hqos->hqos_field0_slabpos,
747 hqos->hqos_field0_slabmask,
748 hqos->hqos_field0_slabshr);
749 u64 pkt_pipe = BITFIELD (pkt_data, hqos->hqos_field1_slabpos,
750 hqos->hqos_field1_slabmask,
751 hqos->hqos_field1_slabshr);
752 u64 pkt_dscp = BITFIELD (pkt_data, hqos->hqos_field2_slabpos,
753 hqos->hqos_field2_slabmask,
754 hqos->hqos_field2_slabshr);
755 u32 pkt_tc = hqos->hqos_tc_table[pkt_dscp & 0x3F] >> 2;
756 u32 pkt_tc_q = hqos->hqos_tc_table[pkt_dscp & 0x3F] & 0x3;
758 u64 pkt_sched = RTE_SCHED_PORT_HIERARCHY (pkt_subport,
764 pkt->hash.sched.lo = pkt_sched & 0xFFFFFFFF;
765 pkt->hash.sched.hi = pkt_sched >> 32;
770 * fd.io coding-style-patch-verification: ON
773 * eval: (c-set-style "gnu")