src/plugins/dpdk/hqos/hqos.c

   1 /*
   2  * Copyright(c) 2016 Intel Corporation. All rights reserved.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 #include <stdio.h>
  16 #include <stdlib.h>
  17 #include <unistd.h>
  18 #include <sys/stat.h>
  19 #include <sys/mount.h>
  20 #include <string.h>
  21 #include <fcntl.h>
  22
  23 #include <vppinfra/vec.h>
  24 #include <vppinfra/error.h>
  25 #include <vppinfra/format.h>
  26 #include <vppinfra/bitmap.h>
  27
  28 #include <vnet/vnet.h>
  29 #include <vnet/ethernet/ethernet.h>
  30 #include <dpdk/device/dpdk.h>
  31
  32 #include <vlib/unix/physmem.h>
  33 #include <vlib/pci/pci.h>
  34 #include <vlibmemory/api.h>
  35 #include <vlibmemory/vl_memory_msg_enum.h>      /* enumerate all vlib messages */
  36
  37 #define vl_typedefs             /* define message structures */
  38 #include <vlibmemory/vl_memory_api_h.h>
  39 #undef vl_typedefs
  40
  41 /* instantiate all the print functions we know about */
  42 #define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__)
  43 #define vl_printfun
  44 #include <vlibmemory/vl_memory_api_h.h>
  45 #undef vl_printfun
  46
  47 #include <dpdk/device/dpdk_priv.h>
  48
  49 /***
  50  *
  51  * HQoS default configuration values
  52  *
  53  ***/
  54
  55 static dpdk_device_config_hqos_t hqos_params_default = {
  56   .hqos_thread_valid = 0,
  57
  58   .swq_size = 4096,
  59   .burst_enq = 256,
  60   .burst_deq = 220,
  61
  62   /*
  63    * Packet field to identify the subport.
  64    *
  65    * Default value: Since only one subport is defined by default (see below:
  66    *     n_subports_per_port = 1), the subport ID is hardcoded to 0.
  67    */
  68   .pktfield0_slabpos = 0,
  69   .pktfield0_slabmask = 0,
  70
  71   /*
  72    * Packet field to identify the pipe.
  73    *
  74    * Default value: Assuming Ethernet/IPv4/UDP packets, UDP payload bits 12 .. 23
  75    */
  76   .pktfield1_slabpos = 40,
  77   .pktfield1_slabmask = 0x0000000FFF000000LLU,
  78
  79   /* Packet field used as index into TC translation table to identify the traffic
  80    *     class and queue.
  81    *
  82    * Default value: Assuming Ethernet/IPv4 packets, IPv4 DSCP field
  83    */
  84   .pktfield2_slabpos = 8,
  85   .pktfield2_slabmask = 0x00000000000000FCLLU,
  86   .tc_table = {
  87                0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
  88                0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
  89                0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
  90                0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
  91                },
  92
  93   /* port */
  94   .port = {
  95            .name = NULL,        /* Set at init */
  96            .socket = 0,         /* Set at init */
  97            .rate = 1250000000,  /* Assuming 10GbE port */
  98            .mtu = 14 + 1500,    /* Assuming Ethernet/IPv4 pkt (Ethernet FCS not included) */
  99            .frame_overhead = RTE_SCHED_FRAME_OVERHEAD_DEFAULT,
 100            .n_subports_per_port = 1,
 101            .n_pipes_per_subport = 4096,
 102            .qsize = {64, 64, 64, 64},
 103            .pipe_profiles = NULL,       /* Set at config */
 104            .n_pipe_profiles = 1,
 105
 106 #ifdef RTE_SCHED_RED
 107            .red_params = {
 108                           /* Traffic Class 0 Colors Green / Yellow / Red */
 109                           [0][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
 110                                     10,.wq_log2 = 9},
 111                           [0][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
 112                                     10,.wq_log2 = 9},
 113                           [0][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
 114                                     10,.wq_log2 = 9},
 115
 116                           /* Traffic Class 1 - Colors Green / Yellow / Red */
 117                           [1][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
 118                                     10,.wq_log2 = 9},
 119                           [1][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
 120                                     10,.wq_log2 = 9},
 121                           [1][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
 122                                     10,.wq_log2 = 9},
 123
 124                           /* Traffic Class 2 - Colors Green / Yellow / Red */
 125                           [2][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
 126                                     10,.wq_log2 = 9},
 127                           [2][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
 128                                     10,.wq_log2 = 9},
 129                           [2][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
 130                                     10,.wq_log2 = 9},
 131
 132                           /* Traffic Class 3 - Colors Green / Yellow / Red */
 133                           [3][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
 134                                     10,.wq_log2 = 9},
 135                           [3][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
 136                                     10,.wq_log2 = 9},
 137                           [3][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
 138                                     10,.wq_log2 = 9}
 139                           },
 140 #endif /* RTE_SCHED_RED */
 141            },
 142 };
 143
 144 static struct rte_sched_subport_params hqos_subport_params_default = {
 145   .tb_rate = 1250000000,        /* 10GbE line rate (measured in bytes/second) */
 146   .tb_size = 1000000,
 147   .tc_rate = {1250000000, 1250000000, 1250000000, 1250000000},
 148   .tc_period = 10,
 149 };
 150
 151 static struct rte_sched_pipe_params hqos_pipe_params_default = {
 152   .tb_rate = 305175,            /* 10GbE line rate divided by 4K pipes */
 153   .tb_size = 1000000,
 154   .tc_rate = {305175, 305175, 305175, 305175},
 155   .tc_period = 40,
 156 #ifdef RTE_SCHED_SUBPORT_TC_OV
 157   .tc_ov_weight = 1,
 158 #endif
 159   .wrr_weights = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
 160 };
 161
 162 /***
 163  *
 164  * HQoS configuration
 165  *
 166  ***/
 167
 168 int
 169 dpdk_hqos_validate_mask (u64 mask, u32 n)
 170 {
 171   int count = __builtin_popcountll (mask);
 172   int pos_lead = sizeof (u64) * 8 - __builtin_clzll (mask);
 173   int pos_trail = __builtin_ctzll (mask);
 174   int count_expected = __builtin_popcount (n - 1);
 175
 176   /* Handle the exceptions */
 177   if (n == 0)
 178     return -1;                  /* Error */
 179
 180   if ((mask == 0) && (n == 1))
 181     return 0;                   /* OK */
 182
 183   if (((mask == 0) && (n != 1)) || ((mask != 0) && (n == 1)))
 184     return -2;                  /* Error */
 185
 186   /* Check that mask is contiguous */
 187   if ((pos_lead - pos_trail) != count)
 188     return -3;                  /* Error */
 189
 190   /* Check that mask contains the expected number of bits set */
 191   if (count != count_expected)
 192     return -4;                  /* Error */
 193
 194   return 0;                     /* OK */
 195 }
 196
 197 void
 198 dpdk_device_config_hqos_pipe_profile_default (dpdk_device_config_hqos_t *
 199                                               hqos, u32 pipe_profile_id)
 200 {
 201   memcpy (&hqos->pipe[pipe_profile_id], &hqos_pipe_params_default,
 202           sizeof (hqos_pipe_params_default));
 203 }
 204
 205 void
 206 dpdk_device_config_hqos_default (dpdk_device_config_hqos_t * hqos)
 207 {
 208   struct rte_sched_subport_params *subport_params;
 209   struct rte_sched_pipe_params *pipe_params;
 210   u32 *pipe_map;
 211   u32 i;
 212
 213   memcpy (hqos, &hqos_params_default, sizeof (hqos_params_default));
 214
 215   /* pipe */
 216   vec_add2 (hqos->pipe, pipe_params, hqos->port.n_pipe_profiles);
 217
 218   for (i = 0; i < vec_len (hqos->pipe); i++)
 219     memcpy (&pipe_params[i],
 220             &hqos_pipe_params_default, sizeof (hqos_pipe_params_default));
 221
 222   hqos->port.pipe_profiles = hqos->pipe;
 223
 224   /* subport */
 225   vec_add2 (hqos->subport, subport_params, hqos->port.n_subports_per_port);
 226
 227   for (i = 0; i < vec_len (hqos->subport); i++)
 228     memcpy (&subport_params[i],
 229             &hqos_subport_params_default,
 230             sizeof (hqos_subport_params_default));
 231
 232   /* pipe profile */
 233   vec_add2 (hqos->pipe_map,
 234             pipe_map,
 235             hqos->port.n_subports_per_port * hqos->port.n_pipes_per_subport);
 236
 237   for (i = 0; i < vec_len (hqos->pipe_map); i++)
 238     pipe_map[i] = 0;
 239 }
 240
 241 /***
 242  *
 243  * HQoS init
 244  *
 245  ***/
 246
 247 clib_error_t *
 248 dpdk_port_setup_hqos (dpdk_device_t * xd, dpdk_device_config_hqos_t * hqos)
 249 {
 250   vlib_thread_main_t *tm = vlib_get_thread_main ();
 251   char name[32];
 252   u32 subport_id, i;
 253   int rv;
 254
 255   /* Detect the set of worker threads */
 256   int worker_thread_first = 0;
 257   int worker_thread_count = 0;
 258
 259   uword *p = hash_get_mem (tm->thread_registrations_by_name, "workers");
 260   vlib_thread_registration_t *tr =
 261     p ? (vlib_thread_registration_t *) p[0] : 0;
 262
 263   if (tr && tr->count > 0)
 264     {
 265       worker_thread_first = tr->first_index;
 266       worker_thread_count = tr->count;
 267     }
 268
 269   /* Allocate the per-thread device data array */
 270   vec_validate_aligned (xd->hqos_wt, tm->n_vlib_mains - 1,
 271                         CLIB_CACHE_LINE_BYTES);
 272   memset (xd->hqos_wt, 0, tm->n_vlib_mains * sizeof (xd->hqos_wt[0]));
 273
 274   vec_validate_aligned (xd->hqos_ht, 0, CLIB_CACHE_LINE_BYTES);
 275   memset (xd->hqos_ht, 0, sizeof (xd->hqos_ht[0]));
 276
 277   /* Allocate space for one SWQ per worker thread in the I/O TX thread data structure */
 278   vec_validate (xd->hqos_ht->swq, worker_thread_count);
 279
 280   /* SWQ */
 281   for (i = 0; i < worker_thread_count + 1; i++)
 282     {
 283       u32 swq_flags = RING_F_SP_ENQ | RING_F_SC_DEQ;
 284
 285       snprintf (name, sizeof (name), "SWQ-worker%u-to-device%u", i,
 286                 xd->device_index);
 287       xd->hqos_ht->swq[i] =
 288         rte_ring_create (name, hqos->swq_size, xd->cpu_socket, swq_flags);
 289       if (xd->hqos_ht->swq[i] == NULL)
 290         return clib_error_return (0,
 291                                   "SWQ-worker%u-to-device%u: rte_ring_create err",
 292                                   i, xd->device_index);
 293     }
 294
 295   /*
 296    * HQoS
 297    */
 298
 299   /* HQoS port */
 300   snprintf (name, sizeof (name), "HQoS%u", xd->device_index);
 301   hqos->port.name = strdup (name);
 302   if (hqos->port.name == NULL)
 303     return clib_error_return (0, "HQoS%u: strdup err", xd->device_index);
 304
 305   hqos->port.socket = rte_eth_dev_socket_id (xd->device_index);
 306   if (hqos->port.socket == SOCKET_ID_ANY)
 307     hqos->port.socket = 0;
 308
 309   xd->hqos_ht->hqos = rte_sched_port_config (&hqos->port);
 310   if (xd->hqos_ht->hqos == NULL)
 311     return clib_error_return (0, "HQoS%u: rte_sched_port_config err",
 312                               xd->device_index);
 313
 314   /* HQoS subport */
 315   for (subport_id = 0; subport_id < hqos->port.n_subports_per_port;
 316        subport_id++)
 317     {
 318       u32 pipe_id;
 319
 320       rv =
 321         rte_sched_subport_config (xd->hqos_ht->hqos, subport_id,
 322                                   &hqos->subport[subport_id]);
 323       if (rv)
 324         return clib_error_return (0,
 325                                   "HQoS%u subport %u: rte_sched_subport_config err (%d)",
 326                                   xd->device_index, subport_id, rv);
 327
 328       /* HQoS pipe */
 329       for (pipe_id = 0; pipe_id < hqos->port.n_pipes_per_subport; pipe_id++)
 330         {
 331           u32 pos = subport_id * hqos->port.n_pipes_per_subport + pipe_id;
 332           u32 profile_id = hqos->pipe_map[pos];
 333
 334           rv =
 335             rte_sched_pipe_config (xd->hqos_ht->hqos, subport_id, pipe_id,
 336                                    profile_id);
 337           if (rv)
 338             return clib_error_return (0,
 339                                       "HQoS%u subport %u pipe %u: rte_sched_pipe_config err (%d)",
 340                                       xd->device_index, subport_id, pipe_id,
 341                                       rv);
 342         }
 343     }
 344
 345   /* Set up per-thread device data for the I/O TX thread */
 346   xd->hqos_ht->hqos_burst_enq = hqos->burst_enq;
 347   xd->hqos_ht->hqos_burst_deq = hqos->burst_deq;
 348   vec_validate (xd->hqos_ht->pkts_enq, 2 * hqos->burst_enq - 1);
 349   vec_validate (xd->hqos_ht->pkts_deq, hqos->burst_deq - 1);
 350   xd->hqos_ht->pkts_enq_len = 0;
 351   xd->hqos_ht->swq_pos = 0;
 352   xd->hqos_ht->flush_count = 0;
 353
 354   /* Set up per-thread device data for each worker thread */
 355   for (i = 0; i < worker_thread_count + 1; i++)
 356     {
 357       u32 tid;
 358       if (i)
 359         tid = worker_thread_first + (i - 1);
 360       else
 361         tid = i;
 362
 363       xd->hqos_wt[tid].swq = xd->hqos_ht->swq[i];
 364       xd->hqos_wt[tid].hqos_field0_slabpos = hqos->pktfield0_slabpos;
 365       xd->hqos_wt[tid].hqos_field0_slabmask = hqos->pktfield0_slabmask;
 366       xd->hqos_wt[tid].hqos_field0_slabshr =
 367         __builtin_ctzll (hqos->pktfield0_slabmask);
 368       xd->hqos_wt[tid].hqos_field1_slabpos = hqos->pktfield1_slabpos;
 369       xd->hqos_wt[tid].hqos_field1_slabmask = hqos->pktfield1_slabmask;
 370       xd->hqos_wt[tid].hqos_field1_slabshr =
 371         __builtin_ctzll (hqos->pktfield1_slabmask);
 372       xd->hqos_wt[tid].hqos_field2_slabpos = hqos->pktfield2_slabpos;
 373       xd->hqos_wt[tid].hqos_field2_slabmask = hqos->pktfield2_slabmask;
 374       xd->hqos_wt[tid].hqos_field2_slabshr =
 375         __builtin_ctzll (hqos->pktfield2_slabmask);
 376       memcpy (xd->hqos_wt[tid].hqos_tc_table, hqos->tc_table,
 377               sizeof (hqos->tc_table));
 378     }
 379
 380   return 0;
 381 }
 382
 383 /***
 384  *
 385  * HQoS run-time
 386  *
 387  ***/
 388 /*
 389  * dpdk_hqos_thread - Contains the main loop of an HQoS thread.
 390  *
 391  * w
 392  *     Information for the current thread
 393  */
 394 static_always_inline void
 395 dpdk_hqos_thread_internal_hqos_dbg_bypass (vlib_main_t * vm)
 396 {
 397   dpdk_main_t *dm = &dpdk_main;
 398   u32 thread_index = vm->thread_index;
 399   u32 dev_pos;
 400
 401   dev_pos = 0;
 402   while (1)
 403     {
 404       vlib_worker_thread_barrier_check ();
 405
 406       u32 n_devs = vec_len (dm->devices_by_hqos_cpu[thread_index]);
 407       if (dev_pos >= n_devs)
 408         dev_pos = 0;
 409
 410       dpdk_device_and_queue_t *dq =
 411         vec_elt_at_index (dm->devices_by_hqos_cpu[thread_index], dev_pos);
 412       dpdk_device_t *xd = vec_elt_at_index (dm->devices, dq->device);
 413
 414       dpdk_device_hqos_per_hqos_thread_t *hqos = xd->hqos_ht;
 415       u32 device_index = xd->device_index;
 416       u16 queue_id = dq->queue_id;
 417
 418       struct rte_mbuf **pkts_enq = hqos->pkts_enq;
 419       u32 pkts_enq_len = hqos->pkts_enq_len;
 420       u32 swq_pos = hqos->swq_pos;
 421       u32 n_swq = vec_len (hqos->swq), i;
 422       u32 flush_count = hqos->flush_count;
 423
 424       for (i = 0; i < n_swq; i++)
 425         {
 426           /* Get current SWQ for this device */
 427           struct rte_ring *swq = hqos->swq[swq_pos];
 428
 429           /* Read SWQ burst to packet buffer of this device */
 430           pkts_enq_len += rte_ring_sc_dequeue_burst (swq,
 431                                                      (void **)
 432                                                      &pkts_enq[pkts_enq_len],
 433                                                      hqos->hqos_burst_enq, 0);
 434
 435           /* Get next SWQ for this device */
 436           swq_pos++;
 437           if (swq_pos >= n_swq)
 438             swq_pos = 0;
 439           hqos->swq_pos = swq_pos;
 440
 441           /* HWQ TX enqueue when burst available */
 442           if (pkts_enq_len >= hqos->hqos_burst_enq)
 443             {
 444               u32 n_pkts = rte_eth_tx_burst (device_index,
 445                                              (uint16_t) queue_id,
 446                                              pkts_enq,
 447                                              (uint16_t) pkts_enq_len);
 448
 449               for (; n_pkts < pkts_enq_len; n_pkts++)
 450                 rte_pktmbuf_free (pkts_enq[n_pkts]);
 451
 452               pkts_enq_len = 0;
 453               flush_count = 0;
 454               break;
 455             }
 456         }
 457       if (pkts_enq_len)
 458         {
 459           flush_count++;
 460           if (PREDICT_FALSE (flush_count == HQOS_FLUSH_COUNT_THRESHOLD))
 461             {
 462               rte_sched_port_enqueue (hqos->hqos, pkts_enq, pkts_enq_len);
 463
 464               pkts_enq_len = 0;
 465               flush_count = 0;
 466             }
 467         }
 468       hqos->pkts_enq_len = pkts_enq_len;
 469       hqos->flush_count = flush_count;
 470
 471       /* Advance to next device */
 472       dev_pos++;
 473     }
 474 }
 475
 476 static_always_inline void
 477 dpdk_hqos_thread_internal (vlib_main_t * vm)
 478 {
 479   dpdk_main_t *dm = &dpdk_main;
 480   u32 thread_index = vm->thread_index;
 481   u32 dev_pos;
 482
 483   dev_pos = 0;
 484   while (1)
 485     {
 486       vlib_worker_thread_barrier_check ();
 487
 488       u32 n_devs = vec_len (dm->devices_by_hqos_cpu[thread_index]);
 489       if (PREDICT_FALSE (n_devs == 0))
 490         {
 491           dev_pos = 0;
 492           continue;
 493         }
 494       if (dev_pos >= n_devs)
 495         dev_pos = 0;
 496
 497       dpdk_device_and_queue_t *dq =
 498         vec_elt_at_index (dm->devices_by_hqos_cpu[thread_index], dev_pos);
 499       dpdk_device_t *xd = vec_elt_at_index (dm->devices, dq->device);
 500
 501       dpdk_device_hqos_per_hqos_thread_t *hqos = xd->hqos_ht;
 502       u32 device_index = xd->device_index;
 503       u16 queue_id = dq->queue_id;
 504
 505       struct rte_mbuf **pkts_enq = hqos->pkts_enq;
 506       struct rte_mbuf **pkts_deq = hqos->pkts_deq;
 507       u32 pkts_enq_len = hqos->pkts_enq_len;
 508       u32 swq_pos = hqos->swq_pos;
 509       u32 n_swq = vec_len (hqos->swq), i;
 510       u32 flush_count = hqos->flush_count;
 511
 512       /*
 513        * SWQ dequeue and HQoS enqueue for current device
 514        */
 515       for (i = 0; i < n_swq; i++)
 516         {
 517           /* Get current SWQ for this device */
 518           struct rte_ring *swq = hqos->swq[swq_pos];
 519
 520           /* Read SWQ burst to packet buffer of this device */
 521           pkts_enq_len += rte_ring_sc_dequeue_burst (swq,
 522                                                      (void **)
 523                                                      &pkts_enq[pkts_enq_len],
 524                                                      hqos->hqos_burst_enq, 0);
 525
 526           /* Get next SWQ for this device */
 527           swq_pos++;
 528           if (swq_pos >= n_swq)
 529             swq_pos = 0;
 530           hqos->swq_pos = swq_pos;
 531
 532           /* HQoS enqueue when burst available */
 533           if (pkts_enq_len >= hqos->hqos_burst_enq)
 534             {
 535               rte_sched_port_enqueue (hqos->hqos, pkts_enq, pkts_enq_len);
 536
 537               pkts_enq_len = 0;
 538               flush_count = 0;
 539               break;
 540             }
 541         }
 542       if (pkts_enq_len)
 543         {
 544           flush_count++;
 545           if (PREDICT_FALSE (flush_count == HQOS_FLUSH_COUNT_THRESHOLD))
 546             {
 547               rte_sched_port_enqueue (hqos->hqos, pkts_enq, pkts_enq_len);
 548
 549               pkts_enq_len = 0;
 550               flush_count = 0;
 551             }
 552         }
 553       hqos->pkts_enq_len = pkts_enq_len;
 554       hqos->flush_count = flush_count;
 555
 556       /*
 557        * HQoS dequeue and HWQ TX enqueue for current device
 558        */
 559       {
 560         u32 pkts_deq_len, n_pkts;
 561
 562         pkts_deq_len = rte_sched_port_dequeue (hqos->hqos,
 563                                                pkts_deq,
 564                                                hqos->hqos_burst_deq);
 565
 566         for (n_pkts = 0; n_pkts < pkts_deq_len;)
 567           n_pkts += rte_eth_tx_burst (device_index,
 568                                       (uint16_t) queue_id,
 569                                       &pkts_deq[n_pkts],
 570                                       (uint16_t) (pkts_deq_len - n_pkts));
 571       }
 572
 573       /* Advance to next device */
 574       dev_pos++;
 575     }
 576 }
 577
 578 void
 579 dpdk_hqos_thread (vlib_worker_thread_t * w)
 580 {
 581   vlib_main_t *vm;
 582   vlib_thread_main_t *tm = vlib_get_thread_main ();
 583   dpdk_main_t *dm = &dpdk_main;
 584
 585   vm = vlib_get_main ();
 586
 587   ASSERT (vm->thread_index == vlib_get_thread_index ());
 588
 589   clib_time_init (&vm->clib_time);
 590   clib_mem_set_heap (w->thread_mheap);
 591
 592   /* Wait until the dpdk init sequence is complete */
 593   while (tm->worker_thread_release == 0)
 594     vlib_worker_thread_barrier_check ();
 595
 596   if (vec_len (dm->devices_by_hqos_cpu[vm->thread_index]) == 0)
 597     return
 598       clib_error
 599       ("current I/O TX thread does not have any devices assigned to it");
 600
 601   if (DPDK_HQOS_DBG_BYPASS)
 602     dpdk_hqos_thread_internal_hqos_dbg_bypass (vm);
 603   else
 604     dpdk_hqos_thread_internal (vm);
 605 }
 606
 607 void
 608 dpdk_hqos_thread_fn (void *arg)
 609 {
 610   vlib_worker_thread_t *w = (vlib_worker_thread_t *) arg;
 611   vlib_worker_thread_init (w);
 612   dpdk_hqos_thread (w);
 613 }
 614
 615 /* *INDENT-OFF* */
 616 VLIB_REGISTER_THREAD (hqos_thread_reg, static) =
 617 {
 618   .name = "hqos-threads",
 619   .short_name = "hqos-threads",
 620   .function = dpdk_hqos_thread_fn,
 621 };
 622 /* *INDENT-ON* */
 623
 624 /*
 625  * HQoS run-time code to be called by the worker threads
 626  */
 627 #define BITFIELD(byte_array, slab_pos, slab_mask, slab_shr)     \
 628 ({                                                              \
 629   u64 slab = *((u64 *) &byte_array[slab_pos]);                  \
 630   u64 val = (rte_be_to_cpu_64(slab) & slab_mask) >> slab_shr;   \
 631   val;                                                          \
 632 })
 633
 634 #define RTE_SCHED_PORT_HIERARCHY(subport, pipe, traffic_class, queue, color) \
 635   ((((u64) (queue)) & 0x3) |                               \
 636   ((((u64) (traffic_class)) & 0x3) << 2) |                 \
 637   ((((u64) (color)) & 0x3) << 4) |                         \
 638   ((((u64) (subport)) & 0xFFFF) << 16) |                   \
 639   ((((u64) (pipe)) & 0xFFFFFFFF) << 32))
 640
 641 void
 642 dpdk_hqos_metadata_set (dpdk_device_hqos_per_worker_thread_t * hqos,
 643                         struct rte_mbuf **pkts, u32 n_pkts)
 644 {
 645   u32 i;
 646
 647   for (i = 0; i < (n_pkts & (~0x3)); i += 4)
 648     {
 649       struct rte_mbuf *pkt0 = pkts[i];
 650       struct rte_mbuf *pkt1 = pkts[i + 1];
 651       struct rte_mbuf *pkt2 = pkts[i + 2];
 652       struct rte_mbuf *pkt3 = pkts[i + 3];
 653
 654       u8 *pkt0_data = rte_pktmbuf_mtod (pkt0, u8 *);
 655       u8 *pkt1_data = rte_pktmbuf_mtod (pkt1, u8 *);
 656       u8 *pkt2_data = rte_pktmbuf_mtod (pkt2, u8 *);
 657       u8 *pkt3_data = rte_pktmbuf_mtod (pkt3, u8 *);
 658
 659       u64 pkt0_subport = BITFIELD (pkt0_data, hqos->hqos_field0_slabpos,
 660                                    hqos->hqos_field0_slabmask,
 661                                    hqos->hqos_field0_slabshr);
 662       u64 pkt0_pipe = BITFIELD (pkt0_data, hqos->hqos_field1_slabpos,
 663                                 hqos->hqos_field1_slabmask,
 664                                 hqos->hqos_field1_slabshr);
 665       u64 pkt0_dscp = BITFIELD (pkt0_data, hqos->hqos_field2_slabpos,
 666                                 hqos->hqos_field2_slabmask,
 667                                 hqos->hqos_field2_slabshr);
 668       u32 pkt0_tc = hqos->hqos_tc_table[pkt0_dscp & 0x3F] >> 2;
 669       u32 pkt0_tc_q = hqos->hqos_tc_table[pkt0_dscp & 0x3F] & 0x3;
 670
 671       u64 pkt1_subport = BITFIELD (pkt1_data, hqos->hqos_field0_slabpos,
 672                                    hqos->hqos_field0_slabmask,
 673                                    hqos->hqos_field0_slabshr);
 674       u64 pkt1_pipe = BITFIELD (pkt1_data, hqos->hqos_field1_slabpos,
 675                                 hqos->hqos_field1_slabmask,
 676                                 hqos->hqos_field1_slabshr);
 677       u64 pkt1_dscp = BITFIELD (pkt1_data, hqos->hqos_field2_slabpos,
 678                                 hqos->hqos_field2_slabmask,
 679                                 hqos->hqos_field2_slabshr);
 680       u32 pkt1_tc = hqos->hqos_tc_table[pkt1_dscp & 0x3F] >> 2;
 681       u32 pkt1_tc_q = hqos->hqos_tc_table[pkt1_dscp & 0x3F] & 0x3;
 682
 683       u64 pkt2_subport = BITFIELD (pkt2_data, hqos->hqos_field0_slabpos,
 684                                    hqos->hqos_field0_slabmask,
 685                                    hqos->hqos_field0_slabshr);
 686       u64 pkt2_pipe = BITFIELD (pkt2_data, hqos->hqos_field1_slabpos,
 687                                 hqos->hqos_field1_slabmask,
 688                                 hqos->hqos_field1_slabshr);
 689       u64 pkt2_dscp = BITFIELD (pkt2_data, hqos->hqos_field2_slabpos,
 690                                 hqos->hqos_field2_slabmask,
 691                                 hqos->hqos_field2_slabshr);
 692       u32 pkt2_tc = hqos->hqos_tc_table[pkt2_dscp & 0x3F] >> 2;
 693       u32 pkt2_tc_q = hqos->hqos_tc_table[pkt2_dscp & 0x3F] & 0x3;
 694
 695       u64 pkt3_subport = BITFIELD (pkt3_data, hqos->hqos_field0_slabpos,
 696                                    hqos->hqos_field0_slabmask,
 697                                    hqos->hqos_field0_slabshr);
 698       u64 pkt3_pipe = BITFIELD (pkt3_data, hqos->hqos_field1_slabpos,
 699                                 hqos->hqos_field1_slabmask,
 700                                 hqos->hqos_field1_slabshr);
 701       u64 pkt3_dscp = BITFIELD (pkt3_data, hqos->hqos_field2_slabpos,
 702                                 hqos->hqos_field2_slabmask,
 703                                 hqos->hqos_field2_slabshr);
 704       u32 pkt3_tc = hqos->hqos_tc_table[pkt3_dscp & 0x3F] >> 2;
 705       u32 pkt3_tc_q = hqos->hqos_tc_table[pkt3_dscp & 0x3F] & 0x3;
 706
 707       u64 pkt0_sched = RTE_SCHED_PORT_HIERARCHY (pkt0_subport,
 708                                                  pkt0_pipe,
 709                                                  pkt0_tc,
 710                                                  pkt0_tc_q,
 711                                                  0);
 712       u64 pkt1_sched = RTE_SCHED_PORT_HIERARCHY (pkt1_subport,
 713                                                  pkt1_pipe,
 714                                                  pkt1_tc,
 715                                                  pkt1_tc_q,
 716                                                  0);
 717       u64 pkt2_sched = RTE_SCHED_PORT_HIERARCHY (pkt2_subport,
 718                                                  pkt2_pipe,
 719                                                  pkt2_tc,
 720                                                  pkt2_tc_q,
 721                                                  0);
 722       u64 pkt3_sched = RTE_SCHED_PORT_HIERARCHY (pkt3_subport,
 723                                                  pkt3_pipe,
 724                                                  pkt3_tc,
 725                                                  pkt3_tc_q,
 726                                                  0);
 727
 728       pkt0->hash.sched.lo = pkt0_sched & 0xFFFFFFFF;
 729       pkt0->hash.sched.hi = pkt0_sched >> 32;
 730       pkt1->hash.sched.lo = pkt1_sched & 0xFFFFFFFF;
 731       pkt1->hash.sched.hi = pkt1_sched >> 32;
 732       pkt2->hash.sched.lo = pkt2_sched & 0xFFFFFFFF;
 733       pkt2->hash.sched.hi = pkt2_sched >> 32;
 734       pkt3->hash.sched.lo = pkt3_sched & 0xFFFFFFFF;
 735       pkt3->hash.sched.hi = pkt3_sched >> 32;
 736     }
 737
 738   for (; i < n_pkts; i++)
 739     {
 740       struct rte_mbuf *pkt = pkts[i];
 741
 742       u8 *pkt_data = rte_pktmbuf_mtod (pkt, u8 *);
 743
 744       u64 pkt_subport = BITFIELD (pkt_data, hqos->hqos_field0_slabpos,
 745                                   hqos->hqos_field0_slabmask,
 746                                   hqos->hqos_field0_slabshr);
 747       u64 pkt_pipe = BITFIELD (pkt_data, hqos->hqos_field1_slabpos,
 748                                hqos->hqos_field1_slabmask,
 749                                hqos->hqos_field1_slabshr);
 750       u64 pkt_dscp = BITFIELD (pkt_data, hqos->hqos_field2_slabpos,
 751                                hqos->hqos_field2_slabmask,
 752                                hqos->hqos_field2_slabshr);
 753       u32 pkt_tc = hqos->hqos_tc_table[pkt_dscp & 0x3F] >> 2;
 754       u32 pkt_tc_q = hqos->hqos_tc_table[pkt_dscp & 0x3F] & 0x3;
 755
 756       u64 pkt_sched = RTE_SCHED_PORT_HIERARCHY (pkt_subport,
 757                                                 pkt_pipe,
 758                                                 pkt_tc,
 759                                                 pkt_tc_q,
 760                                                 0);
 761
 762       pkt->hash.sched.lo = pkt_sched & 0xFFFFFFFF;
 763       pkt->hash.sched.hi = pkt_sched >> 32;
 764     }
 765 }
 766
 767 /*
 768  * fd.io coding-style-patch-verification: ON
 769  *
 770  * Local Variables:
 771  * eval: (c-set-style "gnu")
 772  * End:
 773  */