src/vnet/devices/dpdk/hqos.c

   1 /*
   2  * Copyright(c) 2016 Intel Corporation. All rights reserved.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 #include <stdio.h>
  16 #include <stdlib.h>
  17 #include <unistd.h>
  18 #include <sys/stat.h>
  19 #include <sys/mount.h>
  20 #include <string.h>
  21 #include <fcntl.h>
  22
  23 #include <vppinfra/vec.h>
  24 #include <vppinfra/error.h>
  25 #include <vppinfra/format.h>
  26 #include <vppinfra/bitmap.h>
  27
  28 #include <vnet/vnet.h>
  29 #include <vnet/ethernet/ethernet.h>
  30 #include <vnet/devices/dpdk/dpdk.h>
  31
  32 #include <vlib/unix/physmem.h>
  33 #include <vlib/pci/pci.h>
  34 #include <vlibmemory/api.h>
  35 #include <vlibmemory/vl_memory_msg_enum.h>      /* enumerate all vlib messages */
  36
  37 #define vl_typedefs             /* define message structures */
  38 #include <vlibmemory/vl_memory_api_h.h>
  39 #undef vl_typedefs
  40
  41 /* instantiate all the print functions we know about */
  42 #define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__)
  43 #define vl_printfun
  44 #include <vlibmemory/vl_memory_api_h.h>
  45 #undef vl_printfun
  46
  47 #include "dpdk_priv.h"
  48
  49 dpdk_main_t dpdk_main;
  50
  51 /***
  52  *
  53  * HQoS default configuration values
  54  *
  55  ***/
  56
  57 static dpdk_device_config_hqos_t hqos_params_default = {
  58   .hqos_thread_valid = 0,
  59
  60   .swq_size = 4096,
  61   .burst_enq = 256,
  62   .burst_deq = 220,
  63
  64   /*
  65    * Packet field to identify the subport.
  66    *
  67    * Default value: Since only one subport is defined by default (see below:
  68    *     n_subports_per_port = 1), the subport ID is hardcoded to 0.
  69    */
  70   .pktfield0_slabpos = 0,
  71   .pktfield0_slabmask = 0,
  72
  73   /*
  74    * Packet field to identify the pipe.
  75    *
  76    * Default value: Assuming Ethernet/IPv4/UDP packets, UDP payload bits 12 .. 23
  77    */
  78   .pktfield1_slabpos = 40,
  79   .pktfield1_slabmask = 0x0000000FFF000000LLU,
  80
  81   /* Packet field used as index into TC translation table to identify the traffic
  82    *     class and queue.
  83    *
  84    * Default value: Assuming Ethernet/IPv4 packets, IPv4 DSCP field
  85    */
  86   .pktfield2_slabpos = 8,
  87   .pktfield2_slabmask = 0x00000000000000FCLLU,
  88   .tc_table = {
  89                0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
  90                0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
  91                0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
  92                0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
  93                },
  94
  95   /* port */
  96   .port = {
  97            .name = NULL,        /* Set at init */
  98            .socket = 0,         /* Set at init */
  99            .rate = 1250000000,  /* Assuming 10GbE port */
 100            .mtu = 14 + 1500,    /* Assuming Ethernet/IPv4 pkt (Ethernet FCS not included) */
 101            .frame_overhead = RTE_SCHED_FRAME_OVERHEAD_DEFAULT,
 102            .n_subports_per_port = 1,
 103            .n_pipes_per_subport = 4096,
 104            .qsize = {64, 64, 64, 64},
 105            .pipe_profiles = NULL,       /* Set at config */
 106            .n_pipe_profiles = 1,
 107
 108 #ifdef RTE_SCHED_RED
 109            .red_params = {
 110                           /* Traffic Class 0 Colors Green / Yellow / Red */
 111                           [0][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
 112                                     10,.wq_log2 = 9},
 113                           [0][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
 114                                     10,.wq_log2 = 9},
 115                           [0][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
 116                                     10,.wq_log2 = 9},
 117
 118                           /* Traffic Class 1 - Colors Green / Yellow / Red */
 119                           [1][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
 120                                     10,.wq_log2 = 9},
 121                           [1][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
 122                                     10,.wq_log2 = 9},
 123                           [1][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
 124                                     10,.wq_log2 = 9},
 125
 126                           /* Traffic Class 2 - Colors Green / Yellow / Red */
 127                           [2][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
 128                                     10,.wq_log2 = 9},
 129                           [2][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
 130                                     10,.wq_log2 = 9},
 131                           [2][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
 132                                     10,.wq_log2 = 9},
 133
 134                           /* Traffic Class 3 - Colors Green / Yellow / Red */
 135                           [3][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
 136                                     10,.wq_log2 = 9},
 137                           [3][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
 138                                     10,.wq_log2 = 9},
 139                           [3][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
 140                                     10,.wq_log2 = 9}
 141                           },
 142 #endif /* RTE_SCHED_RED */
 143            },
 144 };
 145
 146 static struct rte_sched_subport_params hqos_subport_params_default = {
 147   .tb_rate = 1250000000,        /* 10GbE line rate (measured in bytes/second) */
 148   .tb_size = 1000000,
 149   .tc_rate = {1250000000, 1250000000, 1250000000, 1250000000},
 150   .tc_period = 10,
 151 };
 152
 153 static struct rte_sched_pipe_params hqos_pipe_params_default = {
 154   .tb_rate = 305175,            /* 10GbE line rate divided by 4K pipes */
 155   .tb_size = 1000000,
 156   .tc_rate = {305175, 305175, 305175, 305175},
 157   .tc_period = 40,
 158 #ifdef RTE_SCHED_SUBPORT_TC_OV
 159   .tc_ov_weight = 1,
 160 #endif
 161   .wrr_weights = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
 162 };
 163
 164 /***
 165  *
 166  * HQoS configuration
 167  *
 168  ***/
 169
 170 int
 171 dpdk_hqos_validate_mask (u64 mask, u32 n)
 172 {
 173   int count = __builtin_popcountll (mask);
 174   int pos_lead = sizeof (u64) * 8 - __builtin_clzll (mask);
 175   int pos_trail = __builtin_ctzll (mask);
 176   int count_expected = __builtin_popcount (n - 1);
 177
 178   /* Handle the exceptions */
 179   if (n == 0)
 180     return -1;                  /* Error */
 181
 182   if ((mask == 0) && (n == 1))
 183     return 0;                   /* OK */
 184
 185   if (((mask == 0) && (n != 1)) || ((mask != 0) && (n == 1)))
 186     return -2;                  /* Error */
 187
 188   /* Check that mask is contiguous */
 189   if ((pos_lead - pos_trail) != count)
 190     return -3;                  /* Error */
 191
 192   /* Check that mask contains the expected number of bits set */
 193   if (count != count_expected)
 194     return -4;                  /* Error */
 195
 196   return 0;                     /* OK */
 197 }
 198
 199 void
 200 dpdk_device_config_hqos_pipe_profile_default (dpdk_device_config_hqos_t *
 201                                               hqos, u32 pipe_profile_id)
 202 {
 203   memcpy (&hqos->pipe[pipe_profile_id], &hqos_pipe_params_default,
 204           sizeof (hqos_pipe_params_default));
 205 }
 206
 207 void
 208 dpdk_device_config_hqos_default (dpdk_device_config_hqos_t * hqos)
 209 {
 210   struct rte_sched_subport_params *subport_params;
 211   struct rte_sched_pipe_params *pipe_params;
 212   u32 *pipe_map;
 213   u32 i;
 214
 215   memcpy (hqos, &hqos_params_default, sizeof (hqos_params_default));
 216
 217   /* pipe */
 218   vec_add2 (hqos->pipe, pipe_params, hqos->port.n_pipe_profiles);
 219
 220   for (i = 0; i < vec_len (hqos->pipe); i++)
 221     memcpy (&pipe_params[i],
 222             &hqos_pipe_params_default, sizeof (hqos_pipe_params_default));
 223
 224   hqos->port.pipe_profiles = hqos->pipe;
 225
 226   /* subport */
 227   vec_add2 (hqos->subport, subport_params, hqos->port.n_subports_per_port);
 228
 229   for (i = 0; i < vec_len (hqos->subport); i++)
 230     memcpy (&subport_params[i],
 231             &hqos_subport_params_default,
 232             sizeof (hqos_subport_params_default));
 233
 234   /* pipe profile */
 235   vec_add2 (hqos->pipe_map,
 236             pipe_map,
 237             hqos->port.n_subports_per_port * hqos->port.n_pipes_per_subport);
 238
 239   for (i = 0; i < vec_len (hqos->pipe_map); i++)
 240     pipe_map[i] = 0;
 241 }
 242
 243 /***
 244  *
 245  * HQoS init
 246  *
 247  ***/
 248
 249 clib_error_t *
 250 dpdk_port_setup_hqos (dpdk_device_t * xd, dpdk_device_config_hqos_t * hqos)
 251 {
 252   vlib_thread_main_t *tm = vlib_get_thread_main ();
 253   char name[32];
 254   u32 subport_id, i;
 255   int rv;
 256
 257   /* Detect the set of worker threads */
 258   int worker_thread_first = 0;
 259   int worker_thread_count = 0;
 260
 261   uword *p = hash_get_mem (tm->thread_registrations_by_name, "workers");
 262   vlib_thread_registration_t *tr =
 263     p ? (vlib_thread_registration_t *) p[0] : 0;
 264
 265   if (tr && tr->count > 0)
 266     {
 267       worker_thread_first = tr->first_index;
 268       worker_thread_count = tr->count;
 269     }
 270
 271   /* Allocate the per-thread device data array */
 272   vec_validate_aligned (xd->hqos_wt, tm->n_vlib_mains - 1,
 273                         CLIB_CACHE_LINE_BYTES);
 274   memset (xd->hqos_wt, 0, tm->n_vlib_mains * sizeof (xd->hqos_wt[0]));
 275
 276   vec_validate_aligned (xd->hqos_ht, 0, CLIB_CACHE_LINE_BYTES);
 277   memset (xd->hqos_ht, 0, sizeof (xd->hqos_ht[0]));
 278
 279   /* Allocate space for one SWQ per worker thread in the I/O TX thread data structure */
 280   vec_validate (xd->hqos_ht->swq, worker_thread_count);
 281
 282   /* SWQ */
 283   for (i = 0; i < worker_thread_count + 1; i++)
 284     {
 285       u32 swq_flags = RING_F_SP_ENQ | RING_F_SC_DEQ;
 286
 287       snprintf (name, sizeof (name), "SWQ-worker%u-to-device%u", i,
 288                 xd->device_index);
 289       xd->hqos_ht->swq[i] =
 290         rte_ring_create (name, hqos->swq_size, xd->cpu_socket, swq_flags);
 291       if (xd->hqos_ht->swq[i] == NULL)
 292         return clib_error_return (0,
 293                                   "SWQ-worker%u-to-device%u: rte_ring_create err",
 294                                   i, xd->device_index);
 295     }
 296
 297   /*
 298    * HQoS
 299    */
 300
 301   /* HQoS port */
 302   snprintf (name, sizeof (name), "HQoS%u", xd->device_index);
 303   hqos->port.name = strdup (name);
 304   if (hqos->port.name == NULL)
 305     return clib_error_return (0, "HQoS%u: strdup err", xd->device_index);
 306
 307   hqos->port.socket = rte_eth_dev_socket_id (xd->device_index);
 308   if (hqos->port.socket == SOCKET_ID_ANY)
 309     hqos->port.socket = 0;
 310
 311   xd->hqos_ht->hqos = rte_sched_port_config (&hqos->port);
 312   if (xd->hqos_ht->hqos == NULL)
 313     return clib_error_return (0, "HQoS%u: rte_sched_port_config err",
 314                               xd->device_index);
 315
 316   /* HQoS subport */
 317   for (subport_id = 0; subport_id < hqos->port.n_subports_per_port;
 318        subport_id++)
 319     {
 320       u32 pipe_id;
 321
 322       rv =
 323         rte_sched_subport_config (xd->hqos_ht->hqos, subport_id,
 324                                   &hqos->subport[subport_id]);
 325       if (rv)
 326         return clib_error_return (0,
 327                                   "HQoS%u subport %u: rte_sched_subport_config err (%d)",
 328                                   xd->device_index, subport_id, rv);
 329
 330       /* HQoS pipe */
 331       for (pipe_id = 0; pipe_id < hqos->port.n_pipes_per_subport; pipe_id++)
 332         {
 333           u32 pos = subport_id * hqos->port.n_pipes_per_subport + pipe_id;
 334           u32 profile_id = hqos->pipe_map[pos];
 335
 336           rv =
 337             rte_sched_pipe_config (xd->hqos_ht->hqos, subport_id, pipe_id,
 338                                    profile_id);
 339           if (rv)
 340             return clib_error_return (0,
 341                                       "HQoS%u subport %u pipe %u: rte_sched_pipe_config err (%d)",
 342                                       xd->device_index, subport_id, pipe_id,
 343                                       rv);
 344         }
 345     }
 346
 347   /* Set up per-thread device data for the I/O TX thread */
 348   xd->hqos_ht->hqos_burst_enq = hqos->burst_enq;
 349   xd->hqos_ht->hqos_burst_deq = hqos->burst_deq;
 350   vec_validate (xd->hqos_ht->pkts_enq, 2 * hqos->burst_enq - 1);
 351   vec_validate (xd->hqos_ht->pkts_deq, hqos->burst_deq - 1);
 352   xd->hqos_ht->pkts_enq_len = 0;
 353   xd->hqos_ht->swq_pos = 0;
 354   xd->hqos_ht->flush_count = 0;
 355
 356   /* Set up per-thread device data for each worker thread */
 357   for (i = 0; i < worker_thread_count + 1; i++)
 358     {
 359       u32 tid;
 360       if (i)
 361         tid = worker_thread_first + (i - 1);
 362       else
 363         tid = i;
 364
 365       xd->hqos_wt[tid].swq = xd->hqos_ht->swq[i];
 366       xd->hqos_wt[tid].hqos_field0_slabpos = hqos->pktfield0_slabpos;
 367       xd->hqos_wt[tid].hqos_field0_slabmask = hqos->pktfield0_slabmask;
 368       xd->hqos_wt[tid].hqos_field0_slabshr =
 369         __builtin_ctzll (hqos->pktfield0_slabmask);
 370       xd->hqos_wt[tid].hqos_field1_slabpos = hqos->pktfield1_slabpos;
 371       xd->hqos_wt[tid].hqos_field1_slabmask = hqos->pktfield1_slabmask;
 372       xd->hqos_wt[tid].hqos_field1_slabshr =
 373         __builtin_ctzll (hqos->pktfield1_slabmask);
 374       xd->hqos_wt[tid].hqos_field2_slabpos = hqos->pktfield2_slabpos;
 375       xd->hqos_wt[tid].hqos_field2_slabmask = hqos->pktfield2_slabmask;
 376       xd->hqos_wt[tid].hqos_field2_slabshr =
 377         __builtin_ctzll (hqos->pktfield2_slabmask);
 378       memcpy (xd->hqos_wt[tid].hqos_tc_table, hqos->tc_table,
 379               sizeof (hqos->tc_table));
 380     }
 381
 382   return 0;
 383 }
 384
 385 /***
 386  *
 387  * HQoS run-time
 388  *
 389  ***/
 390 /*
 391  * dpdk_hqos_thread - Contains the main loop of an HQoS thread.
 392  *
 393  * w
 394  *     Information for the current thread
 395  */
 396 static_always_inline void
 397 dpdk_hqos_thread_internal_hqos_dbg_bypass (vlib_main_t * vm)
 398 {
 399   dpdk_main_t *dm = &dpdk_main;
 400   u32 cpu_index = vm->cpu_index;
 401   u32 dev_pos;
 402
 403   dev_pos = 0;
 404   while (1)
 405     {
 406       vlib_worker_thread_barrier_check ();
 407
 408       u32 n_devs = vec_len (dm->devices_by_hqos_cpu[cpu_index]);
 409       if (dev_pos >= n_devs)
 410         dev_pos = 0;
 411
 412       dpdk_device_and_queue_t *dq =
 413         vec_elt_at_index (dm->devices_by_hqos_cpu[cpu_index], dev_pos);
 414       dpdk_device_t *xd = vec_elt_at_index (dm->devices, dq->device);
 415
 416       dpdk_device_hqos_per_hqos_thread_t *hqos = xd->hqos_ht;
 417       u32 device_index = xd->device_index;
 418       u16 queue_id = dq->queue_id;
 419
 420       struct rte_mbuf **pkts_enq = hqos->pkts_enq;
 421       u32 pkts_enq_len = hqos->pkts_enq_len;
 422       u32 swq_pos = hqos->swq_pos;
 423       u32 n_swq = vec_len (hqos->swq), i;
 424       u32 flush_count = hqos->flush_count;
 425
 426       for (i = 0; i < n_swq; i++)
 427         {
 428           /* Get current SWQ for this device */
 429           struct rte_ring *swq = hqos->swq[swq_pos];
 430
 431           /* Read SWQ burst to packet buffer of this device */
 432           pkts_enq_len += rte_ring_sc_dequeue_burst (swq,
 433                                                      (void **)
 434                                                      &pkts_enq[pkts_enq_len],
 435                                                      hqos->hqos_burst_enq);
 436
 437           /* Get next SWQ for this device */
 438           swq_pos++;
 439           if (swq_pos >= n_swq)
 440             swq_pos = 0;
 441           hqos->swq_pos = swq_pos;
 442
 443           /* HWQ TX enqueue when burst available */
 444           if (pkts_enq_len >= hqos->hqos_burst_enq)
 445             {
 446               u32 n_pkts = rte_eth_tx_burst (device_index,
 447                                              (uint16_t) queue_id,
 448                                              pkts_enq,
 449                                              (uint16_t) pkts_enq_len);
 450
 451               for (; n_pkts < pkts_enq_len; n_pkts++)
 452                 rte_pktmbuf_free (pkts_enq[n_pkts]);
 453
 454               pkts_enq_len = 0;
 455               flush_count = 0;
 456               break;
 457             }
 458         }
 459       if (pkts_enq_len)
 460         {
 461           flush_count++;
 462           if (PREDICT_FALSE (flush_count == HQOS_FLUSH_COUNT_THRESHOLD))
 463             {
 464               rte_sched_port_enqueue (hqos->hqos, pkts_enq, pkts_enq_len);
 465
 466               pkts_enq_len = 0;
 467               flush_count = 0;
 468             }
 469         }
 470       hqos->pkts_enq_len = pkts_enq_len;
 471       hqos->flush_count = flush_count;
 472
 473       /* Advance to next device */
 474       dev_pos++;
 475     }
 476 }
 477
 478 static_always_inline void
 479 dpdk_hqos_thread_internal (vlib_main_t * vm)
 480 {
 481   dpdk_main_t *dm = &dpdk_main;
 482   u32 cpu_index = vm->cpu_index;
 483   u32 dev_pos;
 484
 485   dev_pos = 0;
 486   while (1)
 487     {
 488       vlib_worker_thread_barrier_check ();
 489
 490       u32 n_devs = vec_len (dm->devices_by_hqos_cpu[cpu_index]);
 491       if (PREDICT_FALSE (n_devs == 0))
 492         {
 493           dev_pos = 0;
 494           continue;
 495         }
 496       if (dev_pos >= n_devs)
 497         dev_pos = 0;
 498
 499       dpdk_device_and_queue_t *dq =
 500         vec_elt_at_index (dm->devices_by_hqos_cpu[cpu_index], dev_pos);
 501       dpdk_device_t *xd = vec_elt_at_index (dm->devices, dq->device);
 502
 503       dpdk_device_hqos_per_hqos_thread_t *hqos = xd->hqos_ht;
 504       u32 device_index = xd->device_index;
 505       u16 queue_id = dq->queue_id;
 506
 507       struct rte_mbuf **pkts_enq = hqos->pkts_enq;
 508       struct rte_mbuf **pkts_deq = hqos->pkts_deq;
 509       u32 pkts_enq_len = hqos->pkts_enq_len;
 510       u32 swq_pos = hqos->swq_pos;
 511       u32 n_swq = vec_len (hqos->swq), i;
 512       u32 flush_count = hqos->flush_count;
 513
 514       /*
 515        * SWQ dequeue and HQoS enqueue for current device
 516        */
 517       for (i = 0; i < n_swq; i++)
 518         {
 519           /* Get current SWQ for this device */
 520           struct rte_ring *swq = hqos->swq[swq_pos];
 521
 522           /* Read SWQ burst to packet buffer of this device */
 523           pkts_enq_len += rte_ring_sc_dequeue_burst (swq,
 524                                                      (void **)
 525                                                      &pkts_enq[pkts_enq_len],
 526                                                      hqos->hqos_burst_enq);
 527
 528           /* Get next SWQ for this device */
 529           swq_pos++;
 530           if (swq_pos >= n_swq)
 531             swq_pos = 0;
 532           hqos->swq_pos = swq_pos;
 533
 534           /* HQoS enqueue when burst available */
 535           if (pkts_enq_len >= hqos->hqos_burst_enq)
 536             {
 537               rte_sched_port_enqueue (hqos->hqos, pkts_enq, pkts_enq_len);
 538
 539               pkts_enq_len = 0;
 540               flush_count = 0;
 541               break;
 542             }
 543         }
 544       if (pkts_enq_len)
 545         {
 546           flush_count++;
 547           if (PREDICT_FALSE (flush_count == HQOS_FLUSH_COUNT_THRESHOLD))
 548             {
 549               rte_sched_port_enqueue (hqos->hqos, pkts_enq, pkts_enq_len);
 550
 551               pkts_enq_len = 0;
 552               flush_count = 0;
 553             }
 554         }
 555       hqos->pkts_enq_len = pkts_enq_len;
 556       hqos->flush_count = flush_count;
 557
 558       /*
 559        * HQoS dequeue and HWQ TX enqueue for current device
 560        */
 561       {
 562         u32 pkts_deq_len, n_pkts;
 563
 564         pkts_deq_len = rte_sched_port_dequeue (hqos->hqos,
 565                                                pkts_deq,
 566                                                hqos->hqos_burst_deq);
 567
 568         for (n_pkts = 0; n_pkts < pkts_deq_len;)
 569           n_pkts += rte_eth_tx_burst (device_index,
 570                                       (uint16_t) queue_id,
 571                                       &pkts_deq[n_pkts],
 572                                       (uint16_t) (pkts_deq_len - n_pkts));
 573       }
 574
 575       /* Advance to next device */
 576       dev_pos++;
 577     }
 578 }
 579
 580 void
 581 dpdk_hqos_thread (vlib_worker_thread_t * w)
 582 {
 583   vlib_main_t *vm;
 584   vlib_thread_main_t *tm = vlib_get_thread_main ();
 585   dpdk_main_t *dm = &dpdk_main;
 586
 587   vm = vlib_get_main ();
 588
 589   ASSERT (vm->cpu_index == os_get_cpu_number ());
 590
 591   clib_time_init (&vm->clib_time);
 592   clib_mem_set_heap (w->thread_mheap);
 593
 594   /* Wait until the dpdk init sequence is complete */
 595   while (tm->worker_thread_release == 0)
 596     vlib_worker_thread_barrier_check ();
 597
 598   if (vec_len (dm->devices_by_hqos_cpu[vm->cpu_index]) == 0)
 599     return
 600       clib_error
 601       ("current I/O TX thread does not have any devices assigned to it");
 602
 603   if (DPDK_HQOS_DBG_BYPASS)
 604     dpdk_hqos_thread_internal_hqos_dbg_bypass (vm);
 605   else
 606     dpdk_hqos_thread_internal (vm);
 607 }
 608
 609 void
 610 dpdk_hqos_thread_fn (void *arg)
 611 {
 612   vlib_worker_thread_t *w = (vlib_worker_thread_t *) arg;
 613   vlib_worker_thread_init (w);
 614   dpdk_hqos_thread (w);
 615 }
 616
 617 /* *INDENT-OFF* */
 618 VLIB_REGISTER_THREAD (hqos_thread_reg, static) =
 619 {
 620   .name = "hqos-threads",
 621   .short_name = "hqos-threads",
 622   .function = dpdk_hqos_thread_fn,
 623 };
 624 /* *INDENT-ON* */
 625
 626 /*
 627  * HQoS run-time code to be called by the worker threads
 628  */
 629 #define BITFIELD(byte_array, slab_pos, slab_mask, slab_shr)     \
 630 ({                                                              \
 631   u64 slab = *((u64 *) &byte_array[slab_pos]);                  \
 632   u64 val = (rte_be_to_cpu_64(slab) & slab_mask) >> slab_shr;   \
 633   val;                                                          \
 634 })
 635
 636 #define RTE_SCHED_PORT_HIERARCHY(subport, pipe, traffic_class, queue, color) \
 637   ((((u64) (queue)) & 0x3) |                               \
 638   ((((u64) (traffic_class)) & 0x3) << 2) |                 \
 639   ((((u64) (color)) & 0x3) << 4) |                         \
 640   ((((u64) (subport)) & 0xFFFF) << 16) |                   \
 641   ((((u64) (pipe)) & 0xFFFFFFFF) << 32))
 642
 643 void
 644 dpdk_hqos_metadata_set (dpdk_device_hqos_per_worker_thread_t * hqos,
 645                         struct rte_mbuf **pkts, u32 n_pkts)
 646 {
 647   u32 i;
 648
 649   for (i = 0; i < (n_pkts & (~0x3)); i += 4)
 650     {
 651       struct rte_mbuf *pkt0 = pkts[i];
 652       struct rte_mbuf *pkt1 = pkts[i + 1];
 653       struct rte_mbuf *pkt2 = pkts[i + 2];
 654       struct rte_mbuf *pkt3 = pkts[i + 3];
 655
 656       u8 *pkt0_data = rte_pktmbuf_mtod (pkt0, u8 *);
 657       u8 *pkt1_data = rte_pktmbuf_mtod (pkt1, u8 *);
 658       u8 *pkt2_data = rte_pktmbuf_mtod (pkt2, u8 *);
 659       u8 *pkt3_data = rte_pktmbuf_mtod (pkt3, u8 *);
 660
 661       u64 pkt0_subport = BITFIELD (pkt0_data, hqos->hqos_field0_slabpos,
 662                                    hqos->hqos_field0_slabmask,
 663                                    hqos->hqos_field0_slabshr);
 664       u64 pkt0_pipe = BITFIELD (pkt0_data, hqos->hqos_field1_slabpos,
 665                                 hqos->hqos_field1_slabmask,
 666                                 hqos->hqos_field1_slabshr);
 667       u64 pkt0_dscp = BITFIELD (pkt0_data, hqos->hqos_field2_slabpos,
 668                                 hqos->hqos_field2_slabmask,
 669                                 hqos->hqos_field2_slabshr);
 670       u32 pkt0_tc = hqos->hqos_tc_table[pkt0_dscp & 0x3F] >> 2;
 671       u32 pkt0_tc_q = hqos->hqos_tc_table[pkt0_dscp & 0x3F] & 0x3;
 672
 673       u64 pkt1_subport = BITFIELD (pkt1_data, hqos->hqos_field0_slabpos,
 674                                    hqos->hqos_field0_slabmask,
 675                                    hqos->hqos_field0_slabshr);
 676       u64 pkt1_pipe = BITFIELD (pkt1_data, hqos->hqos_field1_slabpos,
 677                                 hqos->hqos_field1_slabmask,
 678                                 hqos->hqos_field1_slabshr);
 679       u64 pkt1_dscp = BITFIELD (pkt1_data, hqos->hqos_field2_slabpos,
 680                                 hqos->hqos_field2_slabmask,
 681                                 hqos->hqos_field2_slabshr);
 682       u32 pkt1_tc = hqos->hqos_tc_table[pkt1_dscp & 0x3F] >> 2;
 683       u32 pkt1_tc_q = hqos->hqos_tc_table[pkt1_dscp & 0x3F] & 0x3;
 684
 685       u64 pkt2_subport = BITFIELD (pkt2_data, hqos->hqos_field0_slabpos,
 686                                    hqos->hqos_field0_slabmask,
 687                                    hqos->hqos_field0_slabshr);
 688       u64 pkt2_pipe = BITFIELD (pkt2_data, hqos->hqos_field1_slabpos,
 689                                 hqos->hqos_field1_slabmask,
 690                                 hqos->hqos_field1_slabshr);
 691       u64 pkt2_dscp = BITFIELD (pkt2_data, hqos->hqos_field2_slabpos,
 692                                 hqos->hqos_field2_slabmask,
 693                                 hqos->hqos_field2_slabshr);
 694       u32 pkt2_tc = hqos->hqos_tc_table[pkt2_dscp & 0x3F] >> 2;
 695       u32 pkt2_tc_q = hqos->hqos_tc_table[pkt2_dscp & 0x3F] & 0x3;
 696
 697       u64 pkt3_subport = BITFIELD (pkt3_data, hqos->hqos_field0_slabpos,
 698                                    hqos->hqos_field0_slabmask,
 699                                    hqos->hqos_field0_slabshr);
 700       u64 pkt3_pipe = BITFIELD (pkt3_data, hqos->hqos_field1_slabpos,
 701                                 hqos->hqos_field1_slabmask,
 702                                 hqos->hqos_field1_slabshr);
 703       u64 pkt3_dscp = BITFIELD (pkt3_data, hqos->hqos_field2_slabpos,
 704                                 hqos->hqos_field2_slabmask,
 705                                 hqos->hqos_field2_slabshr);
 706       u32 pkt3_tc = hqos->hqos_tc_table[pkt3_dscp & 0x3F] >> 2;
 707       u32 pkt3_tc_q = hqos->hqos_tc_table[pkt3_dscp & 0x3F] & 0x3;
 708
 709       u64 pkt0_sched = RTE_SCHED_PORT_HIERARCHY (pkt0_subport,
 710                                                  pkt0_pipe,
 711                                                  pkt0_tc,
 712                                                  pkt0_tc_q,
 713                                                  0);
 714       u64 pkt1_sched = RTE_SCHED_PORT_HIERARCHY (pkt1_subport,
 715                                                  pkt1_pipe,
 716                                                  pkt1_tc,
 717                                                  pkt1_tc_q,
 718                                                  0);
 719       u64 pkt2_sched = RTE_SCHED_PORT_HIERARCHY (pkt2_subport,
 720                                                  pkt2_pipe,
 721                                                  pkt2_tc,
 722                                                  pkt2_tc_q,
 723                                                  0);
 724       u64 pkt3_sched = RTE_SCHED_PORT_HIERARCHY (pkt3_subport,
 725                                                  pkt3_pipe,
 726                                                  pkt3_tc,
 727                                                  pkt3_tc_q,
 728                                                  0);
 729
 730       pkt0->hash.sched.lo = pkt0_sched & 0xFFFFFFFF;
 731       pkt0->hash.sched.hi = pkt0_sched >> 32;
 732       pkt1->hash.sched.lo = pkt1_sched & 0xFFFFFFFF;
 733       pkt1->hash.sched.hi = pkt1_sched >> 32;
 734       pkt2->hash.sched.lo = pkt2_sched & 0xFFFFFFFF;
 735       pkt2->hash.sched.hi = pkt2_sched >> 32;
 736       pkt3->hash.sched.lo = pkt3_sched & 0xFFFFFFFF;
 737       pkt3->hash.sched.hi = pkt3_sched >> 32;
 738     }
 739
 740   for (; i < n_pkts; i++)
 741     {
 742       struct rte_mbuf *pkt = pkts[i];
 743
 744       u8 *pkt_data = rte_pktmbuf_mtod (pkt, u8 *);
 745
 746       u64 pkt_subport = BITFIELD (pkt_data, hqos->hqos_field0_slabpos,
 747                                   hqos->hqos_field0_slabmask,
 748                                   hqos->hqos_field0_slabshr);
 749       u64 pkt_pipe = BITFIELD (pkt_data, hqos->hqos_field1_slabpos,
 750                                hqos->hqos_field1_slabmask,
 751                                hqos->hqos_field1_slabshr);
 752       u64 pkt_dscp = BITFIELD (pkt_data, hqos->hqos_field2_slabpos,
 753                                hqos->hqos_field2_slabmask,
 754                                hqos->hqos_field2_slabshr);
 755       u32 pkt_tc = hqos->hqos_tc_table[pkt_dscp & 0x3F] >> 2;
 756       u32 pkt_tc_q = hqos->hqos_tc_table[pkt_dscp & 0x3F] & 0x3;
 757
 758       u64 pkt_sched = RTE_SCHED_PORT_HIERARCHY (pkt_subport,
 759                                                 pkt_pipe,
 760                                                 pkt_tc,
 761                                                 pkt_tc_q,
 762                                                 0);
 763
 764       pkt->hash.sched.lo = pkt_sched & 0xFFFFFFFF;
 765       pkt->hash.sched.hi = pkt_sched >> 32;
 766     }
 767 }
 768
 769 /*
 770  * fd.io coding-style-patch-verification: ON
 771  *
 772  * Local Variables:
 773  * eval: (c-set-style "gnu")
 774  * End:
 775  */