vnet/vnet/devices/dpdk/hqos.c

   1 /*
   2  * Copyright(c) 2016 Intel Corporation. All rights reserved.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 #include <stdio.h>
  16 #include <stdlib.h>
  17 #include <unistd.h>
  18 #include <sys/stat.h>
  19 #include <sys/mount.h>
  20 #include <string.h>
  21 #include <fcntl.h>
  22
  23 #include <vppinfra/vec.h>
  24 #include <vppinfra/error.h>
  25 #include <vppinfra/format.h>
  26 #include <vppinfra/bitmap.h>
  27
  28 #include <vnet/vnet.h>
  29 #include <vnet/ethernet/ethernet.h>
  30 #include <vnet/devices/dpdk/dpdk.h>
  31
  32 #include <vlib/unix/physmem.h>
  33 #include <vlib/pci/pci.h>
  34 #include <vlibmemory/api.h>
  35 #include <vlibmemory/vl_memory_msg_enum.h>      /* enumerate all vlib messages */
  36
  37 #define vl_typedefs             /* define message structures */
  38 #include <vlibmemory/vl_memory_api_h.h>
  39 #undef vl_typedefs
  40
  41 /* instantiate all the print functions we know about */
  42 #define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__)
  43 #define vl_printfun
  44 #include <vlibmemory/vl_memory_api_h.h>
  45 #undef vl_printfun
  46
  47 #include "dpdk_priv.h"
  48
  49 dpdk_main_t dpdk_main;
  50
  51 /***
  52  *
  53  * HQoS default configuration values
  54  *
  55  ***/
  56
  57 static dpdk_device_config_hqos_t hqos_params_default = {
  58   .hqos_thread_valid = 0,
  59
  60   .swq_size = 4096,
  61   .burst_enq = 256,
  62   .burst_deq = 220,
  63
  64   /*
  65    * Packet field to identify the subport.
  66    *
  67    * Default value: Since only one subport is defined by default (see below:
  68    *     n_subports_per_port = 1), the subport ID is hardcoded to 0.
  69    */
  70   .pktfield0_slabpos = 0,
  71   .pktfield0_slabmask = 0,
  72
  73   /*
  74    * Packet field to identify the pipe.
  75    *
  76    * Default value: Assuming Ethernet/IPv4/UDP packets, UDP payload bits 12 .. 23
  77    */
  78   .pktfield1_slabpos = 40,
  79   .pktfield1_slabmask = 0x0000000FFF000000LLU,
  80
  81   /* Packet field used as index into TC translation table to identify the traffic
  82    *     class and queue.
  83    *
  84    * Default value: Assuming Ethernet/IPv4 packets, IPv4 DSCP field
  85    */
  86   .pktfield2_slabpos = 8,
  87   .pktfield2_slabmask = 0x00000000000000FCLLU,
  88   .tc_table = {
  89                0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
  90                0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
  91                0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
  92                0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
  93                },
  94
  95   /* port */
  96   .port = {
  97            .name = NULL,        /* Set at init */
  98            .socket = 0,         /* Set at init */
  99            .rate = 1250000000,  /* Assuming 10GbE port */
 100            .mtu = 14 + 1500,    /* Assuming Ethernet/IPv4 pkt (Ethernet FCS not included) */
 101            .frame_overhead = RTE_SCHED_FRAME_OVERHEAD_DEFAULT,
 102            .n_subports_per_port = 1,
 103            .n_pipes_per_subport = 4096,
 104            .qsize = {64, 64, 64, 64},
 105            .pipe_profiles = NULL,       /* Set at config */
 106            .n_pipe_profiles = 1,
 107
 108 #ifdef RTE_SCHED_RED
 109            .red_params = {
 110                           /* Traffic Class 0 Colors Green / Yellow / Red */
 111                           [0][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
 112                                     10,.wq_log2 = 9},
 113                           [0][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
 114                                     10,.wq_log2 = 9},
 115                           [0][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
 116                                     10,.wq_log2 = 9},
 117
 118                           /* Traffic Class 1 - Colors Green / Yellow / Red */
 119                           [1][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
 120                                     10,.wq_log2 = 9},
 121                           [1][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
 122                                     10,.wq_log2 = 9},
 123                           [1][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
 124                                     10,.wq_log2 = 9},
 125
 126                           /* Traffic Class 2 - Colors Green / Yellow / Red */
 127                           [2][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
 128                                     10,.wq_log2 = 9},
 129                           [2][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
 130                                     10,.wq_log2 = 9},
 131                           [2][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
 132                                     10,.wq_log2 = 9},
 133
 134                           /* Traffic Class 3 - Colors Green / Yellow / Red */
 135                           [3][0] = {.min_th = 48,.max_th = 64,.maxp_inv =
 136                                     10,.wq_log2 = 9},
 137                           [3][1] = {.min_th = 40,.max_th = 64,.maxp_inv =
 138                                     10,.wq_log2 = 9},
 139                           [3][2] = {.min_th = 32,.max_th = 64,.maxp_inv =
 140                                     10,.wq_log2 = 9}
 141                           },
 142 #endif /* RTE_SCHED_RED */
 143            },
 144 };
 145
 146 static struct rte_sched_subport_params hqos_subport_params_default = {
 147   .tb_rate = 1250000000,        /* 10GbE line rate (measured in bytes/second) */
 148   .tb_size = 1000000,
 149   .tc_rate = {1250000000, 1250000000, 1250000000, 1250000000},
 150   .tc_period = 10,
 151 };
 152
 153 static struct rte_sched_pipe_params hqos_pipe_params_default = {
 154   .tb_rate = 305175,            /* 10GbE line rate divided by 4K pipes */
 155   .tb_size = 1000000,
 156   .tc_rate = {305175, 305175, 305175, 305175},
 157   .tc_period = 40,
 158 #ifdef RTE_SCHED_SUBPORT_TC_OV
 159   .tc_ov_weight = 1,
 160 #endif
 161   .wrr_weights = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
 162 };
 163
 164 /***
 165  *
 166  * HQoS configuration
 167  *
 168  ***/
 169
 170 int
 171 dpdk_hqos_validate_mask (u64 mask, u32 n)
 172 {
 173   int count = __builtin_popcountll (mask);
 174   int pos_lead = sizeof (u64) * 8 - __builtin_clzll (mask);
 175   int pos_trail = __builtin_ctzll (mask);
 176   int count_expected = __builtin_popcount (n - 1);
 177
 178   /* Handle the exceptions */
 179   if (n == 0)
 180     return -1;                  /* Error */
 181
 182   if ((mask == 0) && (n == 1))
 183     return 0;                   /* OK */
 184
 185   if (((mask == 0) && (n != 1)) || ((mask != 0) && (n == 1)))
 186     return -2;                  /* Error */
 187
 188   /* Check that mask is contiguous */
 189   if ((pos_lead - pos_trail) != count)
 190     return -3;                  /* Error */
 191
 192   /* Check that mask contains the expected number of bits set */
 193   if (count != count_expected)
 194     return -4;                  /* Error */
 195
 196   return 0;                     /* OK */
 197 }
 198
 199 void
 200 dpdk_device_config_hqos_pipe_profile_default (dpdk_device_config_hqos_t *
 201                                               hqos, u32 pipe_profile_id)
 202 {
 203   memcpy (&hqos->pipe[pipe_profile_id], &hqos_pipe_params_default,
 204           sizeof (hqos_pipe_params_default));
 205 }
 206
 207 void
 208 dpdk_device_config_hqos_default (dpdk_device_config_hqos_t * hqos)
 209 {
 210   struct rte_sched_subport_params *subport_params;
 211   struct rte_sched_pipe_params *pipe_params;
 212   u32 *pipe_map;
 213   u32 i;
 214
 215   memcpy (hqos, &hqos_params_default, sizeof (hqos_params_default));
 216
 217   /* pipe */
 218   vec_add2 (hqos->pipe, pipe_params, hqos->port.n_pipe_profiles);
 219
 220   for (i = 0; i < vec_len (hqos->pipe); i++)
 221     memcpy (&pipe_params[i],
 222             &hqos_pipe_params_default, sizeof (hqos_pipe_params_default));
 223
 224   hqos->port.pipe_profiles = hqos->pipe;
 225
 226   /* subport */
 227   vec_add2 (hqos->subport, subport_params, hqos->port.n_subports_per_port);
 228
 229   for (i = 0; i < vec_len (hqos->subport); i++)
 230     memcpy (&subport_params[i],
 231             &hqos_subport_params_default,
 232             sizeof (hqos_subport_params_default));
 233
 234   /* pipe profile */
 235   vec_add2 (hqos->pipe_map,
 236             pipe_map,
 237             hqos->port.n_subports_per_port * hqos->port.n_pipes_per_subport);
 238
 239   for (i = 0; i < vec_len (hqos->pipe_map); i++)
 240     pipe_map[i] = 0;
 241 }
 242
 243 /***
 244  *
 245  * HQoS init
 246  *
 247  ***/
 248
 249 clib_error_t *
 250 dpdk_port_setup_hqos (dpdk_device_t * xd, dpdk_device_config_hqos_t * hqos)
 251 {
 252   vlib_thread_main_t *tm = vlib_get_thread_main ();
 253   char name[32];
 254   u32 subport_id, i;
 255   int rv;
 256
 257   /* Detect the set of worker threads */
 258   int worker_thread_first = 0;
 259   int worker_thread_count = 0;
 260
 261   uword *p = hash_get_mem (tm->thread_registrations_by_name, "workers");
 262   vlib_thread_registration_t *tr =
 263     p ? (vlib_thread_registration_t *) p[0] : 0;
 264
 265   if (tr && tr->count > 0)
 266     {
 267       worker_thread_first = tr->first_index;
 268       worker_thread_count = tr->count;
 269     }
 270
 271   /* Allocate the per-thread device data array */
 272   vec_validate_aligned (xd->hqos_wt, tm->n_vlib_mains - 1,
 273                         CLIB_CACHE_LINE_BYTES);
 274   memset (xd->hqos_wt, 0, tm->n_vlib_mains * sizeof (xd->hqos_wt[0]));
 275
 276   vec_validate_aligned (xd->hqos_ht, 0, CLIB_CACHE_LINE_BYTES);
 277   memset (xd->hqos_ht, 0, sizeof (xd->hqos_ht[0]));
 278
 279   /* Allocate space for one SWQ per worker thread in the I/O TX thread data structure */
 280   vec_validate (xd->hqos_ht->swq, worker_thread_count - 1);
 281
 282   /* SWQ */
 283   for (i = 0; i < worker_thread_count; i++)
 284     {
 285       u32 swq_flags = RING_F_SP_ENQ | RING_F_SC_DEQ;
 286
 287       snprintf (name, sizeof (name), "SWQ-worker%u-to-device%u", i,
 288                 xd->device_index);
 289       xd->hqos_ht->swq[i] =
 290         rte_ring_create (name, hqos->swq_size, xd->cpu_socket, swq_flags);
 291       if (xd->hqos_ht->swq[i] == NULL)
 292         return clib_error_return (0,
 293                                   "SWQ-worker%u-to-device%u: rte_ring_create err",
 294                                   i, xd->device_index);
 295     }
 296
 297   /*
 298    * HQoS
 299    */
 300
 301   /* HQoS port */
 302   snprintf (name, sizeof (name), "HQoS%u", xd->device_index);
 303   hqos->port.name = strdup (name);
 304   if (hqos->port.name == NULL)
 305     return clib_error_return (0, "HQoS%u: strdup err", xd->device_index);
 306
 307   hqos->port.socket = rte_eth_dev_socket_id (xd->device_index);
 308   if (hqos->port.socket == SOCKET_ID_ANY)
 309     hqos->port.socket = 0;
 310
 311   xd->hqos_ht->hqos = rte_sched_port_config (&hqos->port);
 312   if (xd->hqos_ht->hqos == NULL)
 313     return clib_error_return (0, "HQoS%u: rte_sched_port_config err",
 314                               xd->device_index);
 315
 316   /* HQoS subport */
 317   for (subport_id = 0; subport_id < hqos->port.n_subports_per_port;
 318        subport_id++)
 319     {
 320       u32 pipe_id;
 321
 322       rv =
 323         rte_sched_subport_config (xd->hqos_ht->hqos, subport_id,
 324                                   &hqos->subport[subport_id]);
 325       if (rv)
 326         return clib_error_return (0,
 327                                   "HQoS%u subport %u: rte_sched_subport_config err (%d)",
 328                                   xd->device_index, subport_id, rv);
 329
 330       /* HQoS pipe */
 331       for (pipe_id = 0; pipe_id < hqos->port.n_pipes_per_subport; pipe_id++)
 332         {
 333           u32 pos = subport_id * hqos->port.n_pipes_per_subport + pipe_id;
 334           u32 profile_id = hqos->pipe_map[pos];
 335
 336           rv =
 337             rte_sched_pipe_config (xd->hqos_ht->hqos, subport_id, pipe_id,
 338                                    profile_id);
 339           if (rv)
 340             return clib_error_return (0,
 341                                       "HQoS%u subport %u pipe %u: rte_sched_pipe_config err (%d)",
 342                                       xd->device_index, subport_id, pipe_id,
 343                                       rv);
 344         }
 345     }
 346
 347   /* Set up per-thread device data for the I/O TX thread */
 348   xd->hqos_ht->hqos_burst_enq = hqos->burst_enq;
 349   xd->hqos_ht->hqos_burst_deq = hqos->burst_deq;
 350   vec_validate (xd->hqos_ht->pkts_enq, 2 * hqos->burst_enq - 1);
 351   vec_validate (xd->hqos_ht->pkts_deq, hqos->burst_deq - 1);
 352   xd->hqos_ht->pkts_enq_len = 0;
 353   xd->hqos_ht->swq_pos = 0;
 354
 355   /* Set up per-thread device data for each worker thread */
 356   for (i = 0; i < worker_thread_count; i++)
 357     {
 358       u32 tid = worker_thread_first + i;
 359
 360       xd->hqos_wt[tid].swq = xd->hqos_ht->swq[i];
 361       xd->hqos_wt[tid].hqos_field0_slabpos = hqos->pktfield0_slabpos;
 362       xd->hqos_wt[tid].hqos_field0_slabmask = hqos->pktfield0_slabmask;
 363       xd->hqos_wt[tid].hqos_field0_slabshr =
 364         __builtin_ctzll (hqos->pktfield0_slabmask);
 365       xd->hqos_wt[tid].hqos_field1_slabpos = hqos->pktfield1_slabpos;
 366       xd->hqos_wt[tid].hqos_field1_slabmask = hqos->pktfield1_slabmask;
 367       xd->hqos_wt[tid].hqos_field1_slabshr =
 368         __builtin_ctzll (hqos->pktfield1_slabmask);
 369       xd->hqos_wt[tid].hqos_field2_slabpos = hqos->pktfield2_slabpos;
 370       xd->hqos_wt[tid].hqos_field2_slabmask = hqos->pktfield2_slabmask;
 371       xd->hqos_wt[tid].hqos_field2_slabshr =
 372         __builtin_ctzll (hqos->pktfield2_slabmask);
 373       memcpy (xd->hqos_wt[tid].hqos_tc_table, hqos->tc_table,
 374               sizeof (hqos->tc_table));
 375     }
 376
 377   return 0;
 378 }
 379
 380 /***
 381  *
 382  * HQoS run-time
 383  *
 384  ***/
 385 /*
 386  * dpdk_hqos_thread - Contains the main loop of an HQoS thread.
 387  *
 388  * w
 389  *     Information for the current thread
 390  */
 391 static_always_inline void
 392 dpdk_hqos_thread_internal_hqos_dbg_bypass (vlib_main_t * vm)
 393 {
 394   dpdk_main_t *dm = &dpdk_main;
 395   u32 cpu_index = vm->cpu_index;
 396   u32 dev_pos;
 397
 398   dev_pos = 0;
 399   while (1)
 400     {
 401       vlib_worker_thread_barrier_check ();
 402
 403       u32 n_devs = vec_len (dm->devices_by_hqos_cpu[cpu_index]);
 404       if (dev_pos >= n_devs)
 405         dev_pos = 0;
 406
 407       dpdk_device_and_queue_t *dq =
 408         vec_elt_at_index (dm->devices_by_hqos_cpu[cpu_index], dev_pos);
 409       dpdk_device_t *xd = vec_elt_at_index (dm->devices, dq->device);
 410
 411       dpdk_device_hqos_per_hqos_thread_t *hqos = xd->hqos_ht;
 412       u32 device_index = xd->device_index;
 413       u16 queue_id = dq->queue_id;
 414
 415       struct rte_mbuf **pkts_enq = hqos->pkts_enq;
 416       u32 pkts_enq_len = hqos->pkts_enq_len;
 417       u32 swq_pos = hqos->swq_pos;
 418       u32 n_swq = vec_len (hqos->swq), i;
 419
 420       for (i = 0; i < n_swq; i++)
 421         {
 422           /* Get current SWQ for this device */
 423           struct rte_ring *swq = hqos->swq[swq_pos];
 424
 425           /* Read SWQ burst to packet buffer of this device */
 426           pkts_enq_len += rte_ring_sc_dequeue_burst (swq,
 427                                                      (void **)
 428                                                      &pkts_enq[pkts_enq_len],
 429                                                      hqos->hqos_burst_enq);
 430
 431           /* Get next SWQ for this device */
 432           swq_pos++;
 433           if (swq_pos >= n_swq)
 434             swq_pos = 0;
 435           hqos->swq_pos = swq_pos;
 436
 437           /* HWQ TX enqueue when burst available */
 438           if (pkts_enq_len >= hqos->hqos_burst_enq)
 439             {
 440               u32 n_pkts = rte_eth_tx_burst (device_index,
 441                                              (uint16_t) queue_id,
 442                                              pkts_enq,
 443                                              (uint16_t) pkts_enq_len);
 444
 445               for (; n_pkts < pkts_enq_len; n_pkts++)
 446                 rte_pktmbuf_free (pkts_enq[n_pkts]);
 447
 448               pkts_enq_len = 0;
 449               break;
 450             }
 451         }
 452       hqos->pkts_enq_len = pkts_enq_len;
 453
 454       /* Advance to next device */
 455       dev_pos++;
 456     }
 457 }
 458
 459 static_always_inline void
 460 dpdk_hqos_thread_internal (vlib_main_t * vm)
 461 {
 462   dpdk_main_t *dm = &dpdk_main;
 463   u32 cpu_index = vm->cpu_index;
 464   u32 dev_pos;
 465
 466   dev_pos = 0;
 467   while (1)
 468     {
 469       vlib_worker_thread_barrier_check ();
 470
 471       u32 n_devs = vec_len (dm->devices_by_hqos_cpu[cpu_index]);
 472       if (PREDICT_FALSE (n_devs == 0))
 473         {
 474           dev_pos = 0;
 475           continue;
 476         }
 477       if (dev_pos >= n_devs)
 478         dev_pos = 0;
 479
 480       dpdk_device_and_queue_t *dq =
 481         vec_elt_at_index (dm->devices_by_hqos_cpu[cpu_index], dev_pos);
 482       dpdk_device_t *xd = vec_elt_at_index (dm->devices, dq->device);
 483
 484       dpdk_device_hqos_per_hqos_thread_t *hqos = xd->hqos_ht;
 485       u32 device_index = xd->device_index;
 486       u16 queue_id = dq->queue_id;
 487
 488       struct rte_mbuf **pkts_enq = hqos->pkts_enq;
 489       struct rte_mbuf **pkts_deq = hqos->pkts_deq;
 490       u32 pkts_enq_len = hqos->pkts_enq_len;
 491       u32 swq_pos = hqos->swq_pos;
 492       u32 n_swq = vec_len (hqos->swq), i;
 493
 494       /*
 495        * SWQ dequeue and HQoS enqueue for current device
 496        */
 497       for (i = 0; i < n_swq; i++)
 498         {
 499           /* Get current SWQ for this device */
 500           struct rte_ring *swq = hqos->swq[swq_pos];
 501
 502           /* Read SWQ burst to packet buffer of this device */
 503           pkts_enq_len += rte_ring_sc_dequeue_burst (swq,
 504                                                      (void **)
 505                                                      &pkts_enq[pkts_enq_len],
 506                                                      hqos->hqos_burst_enq);
 507
 508           /* Get next SWQ for this device */
 509           swq_pos++;
 510           if (swq_pos >= n_swq)
 511             swq_pos = 0;
 512           hqos->swq_pos = swq_pos;
 513
 514           /* HQoS enqueue when burst available */
 515           if (pkts_enq_len >= hqos->hqos_burst_enq)
 516             {
 517               rte_sched_port_enqueue (hqos->hqos, pkts_enq, pkts_enq_len);
 518
 519               pkts_enq_len = 0;
 520               break;
 521             }
 522         }
 523       hqos->pkts_enq_len = pkts_enq_len;
 524
 525       /*
 526        * HQoS dequeue and HWQ TX enqueue for current device
 527        */
 528       {
 529         u32 pkts_deq_len, n_pkts;
 530
 531         pkts_deq_len = rte_sched_port_dequeue (hqos->hqos,
 532                                                pkts_deq,
 533                                                hqos->hqos_burst_deq);
 534
 535         for (n_pkts = 0; n_pkts < pkts_deq_len;)
 536           n_pkts += rte_eth_tx_burst (device_index,
 537                                       (uint16_t) queue_id,
 538                                       &pkts_deq[n_pkts],
 539                                       (uint16_t) (pkts_deq_len - n_pkts));
 540       }
 541
 542       /* Advance to next device */
 543       dev_pos++;
 544     }
 545 }
 546
 547 void
 548 dpdk_hqos_thread (vlib_worker_thread_t * w)
 549 {
 550   vlib_main_t *vm;
 551   vlib_thread_main_t *tm = vlib_get_thread_main ();
 552   dpdk_main_t *dm = &dpdk_main;
 553
 554   vm = vlib_get_main ();
 555
 556   ASSERT (vm->cpu_index == os_get_cpu_number ());
 557
 558   clib_time_init (&vm->clib_time);
 559   clib_mem_set_heap (w->thread_mheap);
 560
 561   /* Wait until the dpdk init sequence is complete */
 562   while (tm->worker_thread_release == 0)
 563     vlib_worker_thread_barrier_check ();
 564
 565   if (vec_len (dm->devices_by_hqos_cpu[vm->cpu_index]) == 0)
 566     return
 567       clib_error
 568       ("current I/O TX thread does not have any devices assigned to it");
 569
 570   if (DPDK_HQOS_DBG_BYPASS)
 571     dpdk_hqos_thread_internal_hqos_dbg_bypass (vm);
 572   else
 573     dpdk_hqos_thread_internal (vm);
 574 }
 575
 576 void
 577 dpdk_hqos_thread_fn (void *arg)
 578 {
 579   vlib_worker_thread_t *w = (vlib_worker_thread_t *) arg;
 580   vlib_worker_thread_init (w);
 581   dpdk_hqos_thread (w);
 582 }
 583
 584 /* *INDENT-OFF* */
 585 VLIB_REGISTER_THREAD (hqos_thread_reg, static) =
 586 {
 587   .name = "hqos-threads",
 588   .short_name = "hqos-threads",
 589   .function = dpdk_hqos_thread_fn,
 590 };
 591 /* *INDENT-ON* */
 592
 593 /*
 594  * HQoS run-time code to be called by the worker threads
 595  */
 596 #define BITFIELD(byte_array, slab_pos, slab_mask, slab_shr)     \
 597 ({                                                              \
 598   u64 slab = *((u64 *) &byte_array[slab_pos]);                  \
 599   u64 val = (rte_be_to_cpu_64(slab) & slab_mask) >> slab_shr;   \
 600   val;                                                          \
 601 })
 602
 603 #define RTE_SCHED_PORT_HIERARCHY(subport, pipe, traffic_class, queue, color) \
 604   ((((u64) (queue)) & 0x3) |                               \
 605   ((((u64) (traffic_class)) & 0x3) << 2) |                 \
 606   ((((u64) (color)) & 0x3) << 4) |                         \
 607   ((((u64) (subport)) & 0xFFFF) << 16) |                   \
 608   ((((u64) (pipe)) & 0xFFFFFFFF) << 32))
 609
 610 void
 611 dpdk_hqos_metadata_set (dpdk_device_hqos_per_worker_thread_t * hqos,
 612                         struct rte_mbuf **pkts, u32 n_pkts)
 613 {
 614   u32 i;
 615
 616   for (i = 0; i < (n_pkts & (~0x3)); i += 4)
 617     {
 618       struct rte_mbuf *pkt0 = pkts[i];
 619       struct rte_mbuf *pkt1 = pkts[i + 1];
 620       struct rte_mbuf *pkt2 = pkts[i + 2];
 621       struct rte_mbuf *pkt3 = pkts[i + 3];
 622
 623       u8 *pkt0_data = rte_pktmbuf_mtod (pkt0, u8 *);
 624       u8 *pkt1_data = rte_pktmbuf_mtod (pkt1, u8 *);
 625       u8 *pkt2_data = rte_pktmbuf_mtod (pkt2, u8 *);
 626       u8 *pkt3_data = rte_pktmbuf_mtod (pkt3, u8 *);
 627
 628       u64 pkt0_subport = BITFIELD (pkt0_data, hqos->hqos_field0_slabpos,
 629                                    hqos->hqos_field0_slabmask,
 630                                    hqos->hqos_field0_slabshr);
 631       u64 pkt0_pipe = BITFIELD (pkt0_data, hqos->hqos_field1_slabpos,
 632                                 hqos->hqos_field1_slabmask,
 633                                 hqos->hqos_field1_slabshr);
 634       u64 pkt0_dscp = BITFIELD (pkt0_data, hqos->hqos_field2_slabpos,
 635                                 hqos->hqos_field2_slabmask,
 636                                 hqos->hqos_field2_slabshr);
 637       u32 pkt0_tc = hqos->hqos_tc_table[pkt0_dscp & 0x3F] >> 2;
 638       u32 pkt0_tc_q = hqos->hqos_tc_table[pkt0_dscp & 0x3F] & 0x3;
 639
 640       u64 pkt1_subport = BITFIELD (pkt1_data, hqos->hqos_field0_slabpos,
 641                                    hqos->hqos_field0_slabmask,
 642                                    hqos->hqos_field0_slabshr);
 643       u64 pkt1_pipe = BITFIELD (pkt1_data, hqos->hqos_field1_slabpos,
 644                                 hqos->hqos_field1_slabmask,
 645                                 hqos->hqos_field1_slabshr);
 646       u64 pkt1_dscp = BITFIELD (pkt1_data, hqos->hqos_field2_slabpos,
 647                                 hqos->hqos_field2_slabmask,
 648                                 hqos->hqos_field2_slabshr);
 649       u32 pkt1_tc = hqos->hqos_tc_table[pkt1_dscp & 0x3F] >> 2;
 650       u32 pkt1_tc_q = hqos->hqos_tc_table[pkt1_dscp & 0x3F] & 0x3;
 651
 652       u64 pkt2_subport = BITFIELD (pkt2_data, hqos->hqos_field0_slabpos,
 653                                    hqos->hqos_field0_slabmask,
 654                                    hqos->hqos_field0_slabshr);
 655       u64 pkt2_pipe = BITFIELD (pkt2_data, hqos->hqos_field1_slabpos,
 656                                 hqos->hqos_field1_slabmask,
 657                                 hqos->hqos_field1_slabshr);
 658       u64 pkt2_dscp = BITFIELD (pkt2_data, hqos->hqos_field2_slabpos,
 659                                 hqos->hqos_field2_slabmask,
 660                                 hqos->hqos_field2_slabshr);
 661       u32 pkt2_tc = hqos->hqos_tc_table[pkt2_dscp & 0x3F] >> 2;
 662       u32 pkt2_tc_q = hqos->hqos_tc_table[pkt2_dscp & 0x3F] & 0x3;
 663
 664       u64 pkt3_subport = BITFIELD (pkt3_data, hqos->hqos_field0_slabpos,
 665                                    hqos->hqos_field0_slabmask,
 666                                    hqos->hqos_field0_slabshr);
 667       u64 pkt3_pipe = BITFIELD (pkt3_data, hqos->hqos_field1_slabpos,
 668                                 hqos->hqos_field1_slabmask,
 669                                 hqos->hqos_field1_slabshr);
 670       u64 pkt3_dscp = BITFIELD (pkt3_data, hqos->hqos_field2_slabpos,
 671                                 hqos->hqos_field2_slabmask,
 672                                 hqos->hqos_field2_slabshr);
 673       u32 pkt3_tc = hqos->hqos_tc_table[pkt3_dscp & 0x3F] >> 2;
 674       u32 pkt3_tc_q = hqos->hqos_tc_table[pkt3_dscp & 0x3F] & 0x3;
 675
 676       u64 pkt0_sched = RTE_SCHED_PORT_HIERARCHY (pkt0_subport,
 677                                                  pkt0_pipe,
 678                                                  pkt0_tc,
 679                                                  pkt0_tc_q,
 680                                                  0);
 681       u64 pkt1_sched = RTE_SCHED_PORT_HIERARCHY (pkt1_subport,
 682                                                  pkt1_pipe,
 683                                                  pkt1_tc,
 684                                                  pkt1_tc_q,
 685                                                  0);
 686       u64 pkt2_sched = RTE_SCHED_PORT_HIERARCHY (pkt2_subport,
 687                                                  pkt2_pipe,
 688                                                  pkt2_tc,
 689                                                  pkt2_tc_q,
 690                                                  0);
 691       u64 pkt3_sched = RTE_SCHED_PORT_HIERARCHY (pkt3_subport,
 692                                                  pkt3_pipe,
 693                                                  pkt3_tc,
 694                                                  pkt3_tc_q,
 695                                                  0);
 696
 697       pkt0->hash.sched.lo = pkt0_sched & 0xFFFFFFFF;
 698       pkt0->hash.sched.hi = pkt0_sched >> 32;
 699       pkt1->hash.sched.lo = pkt1_sched & 0xFFFFFFFF;
 700       pkt1->hash.sched.hi = pkt1_sched >> 32;
 701       pkt2->hash.sched.lo = pkt2_sched & 0xFFFFFFFF;
 702       pkt2->hash.sched.hi = pkt2_sched >> 32;
 703       pkt3->hash.sched.lo = pkt3_sched & 0xFFFFFFFF;
 704       pkt3->hash.sched.hi = pkt3_sched >> 32;
 705     }
 706
 707   for (; i < n_pkts; i++)
 708     {
 709       struct rte_mbuf *pkt = pkts[i];
 710
 711       u8 *pkt_data = rte_pktmbuf_mtod (pkt, u8 *);
 712
 713       u64 pkt_subport = BITFIELD (pkt_data, hqos->hqos_field0_slabpos,
 714                                   hqos->hqos_field0_slabmask,
 715                                   hqos->hqos_field0_slabshr);
 716       u64 pkt_pipe = BITFIELD (pkt_data, hqos->hqos_field1_slabpos,
 717                                hqos->hqos_field1_slabmask,
 718                                hqos->hqos_field1_slabshr);
 719       u64 pkt_dscp = BITFIELD (pkt_data, hqos->hqos_field2_slabpos,
 720                                hqos->hqos_field2_slabmask,
 721                                hqos->hqos_field2_slabshr);
 722       u32 pkt_tc = hqos->hqos_tc_table[pkt_dscp & 0x3F] >> 2;
 723       u32 pkt_tc_q = hqos->hqos_tc_table[pkt_dscp & 0x3F] & 0x3;
 724
 725       u64 pkt_sched = RTE_SCHED_PORT_HIERARCHY (pkt_subport,
 726                                                 pkt_pipe,
 727                                                 pkt_tc,
 728                                                 pkt_tc_q,
 729                                                 0);
 730
 731       pkt->hash.sched.lo = pkt_sched & 0xFFFFFFFF;
 732       pkt->hash.sched.hi = pkt_sched >> 32;
 733     }
 734 }
 735
 736 /*
 737  * fd.io coding-style-patch-verification: ON
 738  *
 739  * Local Variables:
 740  * eval: (c-set-style "gnu")
 741  * End:
 742  */