drivers/crypto/scheduler/scheduler_pkt_size_distr.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2017 Intel Corporation. All rights reserved.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of Intel Corporation nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <rte_cryptodev.h>
  34 #include <rte_malloc.h>
  35
  36 #include "rte_cryptodev_scheduler_operations.h"
  37 #include "scheduler_pmd_private.h"
  38
  39 #define DEF_PKT_SIZE_THRESHOLD                  (0xffffff80)
  40 #define SLAVE_IDX_SWITCH_MASK                   (0x01)
  41 #define PRIMARY_SLAVE_IDX                       0
  42 #define SECONDARY_SLAVE_IDX                     1
  43 #define NB_PKT_SIZE_SLAVES                      2
  44
  45 /** pkt size based scheduler context */
  46 struct psd_scheduler_ctx {
  47         uint32_t threshold;
  48 };
  49
  50 /** pkt size based scheduler queue pair context */
  51 struct psd_scheduler_qp_ctx {
  52         struct scheduler_slave primary_slave;
  53         struct scheduler_slave secondary_slave;
  54         uint32_t threshold;
  55         uint8_t deq_idx;
  56 } __rte_cache_aligned;
  57
  58 /** scheduling operation variables' wrapping */
  59 struct psd_schedule_op {
  60         uint8_t slave_idx;
  61         uint16_t pos;
  62 };
  63
  64 static uint16_t
  65 schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)
  66 {
  67         struct scheduler_qp_ctx *qp_ctx = qp;
  68         struct psd_scheduler_qp_ctx *psd_qp_ctx = qp_ctx->private_qp_ctx;
  69         struct rte_crypto_op *sched_ops[NB_PKT_SIZE_SLAVES][nb_ops];
  70         uint32_t in_flight_ops[NB_PKT_SIZE_SLAVES] = {
  71                         psd_qp_ctx->primary_slave.nb_inflight_cops,
  72                         psd_qp_ctx->secondary_slave.nb_inflight_cops
  73         };
  74         struct psd_schedule_op enq_ops[NB_PKT_SIZE_SLAVES] = {
  75                 {PRIMARY_SLAVE_IDX, 0}, {SECONDARY_SLAVE_IDX, 0}
  76         };
  77         struct psd_schedule_op *p_enq_op;
  78         uint16_t i, processed_ops_pri = 0, processed_ops_sec = 0;
  79         uint32_t job_len;
  80
  81         if (unlikely(nb_ops == 0))
  82                 return 0;
  83
  84         for (i = 0; i < nb_ops && i < 4; i++) {
  85                 rte_prefetch0(ops[i]->sym);
  86                 rte_prefetch0(ops[i]->sym->session);
  87         }
  88
  89         for (i = 0; (i < (nb_ops - 8)) && (nb_ops > 8); i += 4) {
  90                 rte_prefetch0(ops[i + 4]->sym);
  91                 rte_prefetch0(ops[i + 4]->sym->session);
  92                 rte_prefetch0(ops[i + 5]->sym);
  93                 rte_prefetch0(ops[i + 5]->sym->session);
  94                 rte_prefetch0(ops[i + 6]->sym);
  95                 rte_prefetch0(ops[i + 6]->sym->session);
  96                 rte_prefetch0(ops[i + 7]->sym);
  97                 rte_prefetch0(ops[i + 7]->sym->session);
  98
  99                 /* job_len is initialized as cipher data length, once
 100                  * it is 0, equals to auth data length
 101                  */
 102                 job_len = ops[i]->sym->cipher.data.length;
 103                 job_len += (ops[i]->sym->cipher.data.length == 0) *
 104                                 ops[i]->sym->auth.data.length;
 105                 /* decide the target op based on the job length */
 106                 p_enq_op = &enq_ops[!(job_len & psd_qp_ctx->threshold)];
 107
 108                 /* stop schedule cops before the queue is full, this shall
 109                  * prevent the failed enqueue
 110                  */
 111                 if (p_enq_op->pos + in_flight_ops[p_enq_op->slave_idx] ==
 112                                 qp_ctx->max_nb_objs) {
 113                         i = nb_ops;
 114                         break;
 115                 }
 116
 117                 sched_ops[p_enq_op->slave_idx][p_enq_op->pos] = ops[i];
 118                 p_enq_op->pos++;
 119
 120                 job_len = ops[i+1]->sym->cipher.data.length;
 121                 job_len += (ops[i+1]->sym->cipher.data.length == 0) *
 122                                 ops[i+1]->sym->auth.data.length;
 123                 p_enq_op = &enq_ops[!(job_len & psd_qp_ctx->threshold)];
 124
 125                 if (p_enq_op->pos + in_flight_ops[p_enq_op->slave_idx] ==
 126                                 qp_ctx->max_nb_objs) {
 127                         i = nb_ops;
 128                         break;
 129                 }
 130
 131                 sched_ops[p_enq_op->slave_idx][p_enq_op->pos] = ops[i+1];
 132                 p_enq_op->pos++;
 133
 134                 job_len = ops[i+2]->sym->cipher.data.length;
 135                 job_len += (ops[i+2]->sym->cipher.data.length == 0) *
 136                                 ops[i+2]->sym->auth.data.length;
 137                 p_enq_op = &enq_ops[!(job_len & psd_qp_ctx->threshold)];
 138
 139                 if (p_enq_op->pos + in_flight_ops[p_enq_op->slave_idx] ==
 140                                 qp_ctx->max_nb_objs) {
 141                         i = nb_ops;
 142                         break;
 143                 }
 144
 145                 sched_ops[p_enq_op->slave_idx][p_enq_op->pos] = ops[i+2];
 146                 p_enq_op->pos++;
 147
 148                 job_len = ops[i+3]->sym->cipher.data.length;
 149                 job_len += (ops[i+3]->sym->cipher.data.length == 0) *
 150                                 ops[i+3]->sym->auth.data.length;
 151                 p_enq_op = &enq_ops[!(job_len & psd_qp_ctx->threshold)];
 152
 153                 if (p_enq_op->pos + in_flight_ops[p_enq_op->slave_idx] ==
 154                                 qp_ctx->max_nb_objs) {
 155                         i = nb_ops;
 156                         break;
 157                 }
 158
 159                 sched_ops[p_enq_op->slave_idx][p_enq_op->pos] = ops[i+3];
 160                 p_enq_op->pos++;
 161         }
 162
 163         for (; i < nb_ops; i++) {
 164                 job_len = ops[i]->sym->cipher.data.length;
 165                 job_len += (ops[i]->sym->cipher.data.length == 0) *
 166                                 ops[i]->sym->auth.data.length;
 167                 p_enq_op = &enq_ops[!(job_len & psd_qp_ctx->threshold)];
 168
 169                 if (p_enq_op->pos + in_flight_ops[p_enq_op->slave_idx] ==
 170                                 qp_ctx->max_nb_objs) {
 171                         i = nb_ops;
 172                         break;
 173                 }
 174
 175                 sched_ops[p_enq_op->slave_idx][p_enq_op->pos] = ops[i];
 176                 p_enq_op->pos++;
 177         }
 178
 179         processed_ops_pri = rte_cryptodev_enqueue_burst(
 180                         psd_qp_ctx->primary_slave.dev_id,
 181                         psd_qp_ctx->primary_slave.qp_id,
 182                         sched_ops[PRIMARY_SLAVE_IDX],
 183                         enq_ops[PRIMARY_SLAVE_IDX].pos);
 184         /* enqueue shall not fail as the slave queue is monitored */
 185         RTE_ASSERT(processed_ops_pri == enq_ops[PRIMARY_SLAVE_IDX].pos);
 186
 187         psd_qp_ctx->primary_slave.nb_inflight_cops += processed_ops_pri;
 188
 189         processed_ops_sec = rte_cryptodev_enqueue_burst(
 190                         psd_qp_ctx->secondary_slave.dev_id,
 191                         psd_qp_ctx->secondary_slave.qp_id,
 192                         sched_ops[SECONDARY_SLAVE_IDX],
 193                         enq_ops[SECONDARY_SLAVE_IDX].pos);
 194         RTE_ASSERT(processed_ops_sec == enq_ops[SECONDARY_SLAVE_IDX].pos);
 195
 196         psd_qp_ctx->secondary_slave.nb_inflight_cops += processed_ops_sec;
 197
 198         return processed_ops_pri + processed_ops_sec;
 199 }
 200
 201 static uint16_t
 202 schedule_enqueue_ordering(void *qp, struct rte_crypto_op **ops,
 203                 uint16_t nb_ops)
 204 {
 205         struct rte_ring *order_ring =
 206                         ((struct scheduler_qp_ctx *)qp)->order_ring;
 207         uint16_t nb_ops_to_enq = get_max_enqueue_order_count(order_ring,
 208                         nb_ops);
 209         uint16_t nb_ops_enqd = schedule_enqueue(qp, ops,
 210                         nb_ops_to_enq);
 211
 212         scheduler_order_insert(order_ring, ops, nb_ops_enqd);
 213
 214         return nb_ops_enqd;
 215 }
 216
 217 static uint16_t
 218 schedule_dequeue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)
 219 {
 220         struct psd_scheduler_qp_ctx *qp_ctx =
 221                         ((struct scheduler_qp_ctx *)qp)->private_qp_ctx;
 222         struct scheduler_slave *slaves[NB_PKT_SIZE_SLAVES] = {
 223                         &qp_ctx->primary_slave, &qp_ctx->secondary_slave};
 224         struct scheduler_slave *slave = slaves[qp_ctx->deq_idx];
 225         uint16_t nb_deq_ops_pri = 0, nb_deq_ops_sec = 0;
 226
 227         if (slave->nb_inflight_cops) {
 228                 nb_deq_ops_pri = rte_cryptodev_dequeue_burst(slave->dev_id,
 229                         slave->qp_id, ops, nb_ops);
 230                 slave->nb_inflight_cops -= nb_deq_ops_pri;
 231         }
 232
 233         qp_ctx->deq_idx = (~qp_ctx->deq_idx) & SLAVE_IDX_SWITCH_MASK;
 234
 235         if (nb_deq_ops_pri == nb_ops)
 236                 return nb_deq_ops_pri;
 237
 238         slave = slaves[qp_ctx->deq_idx];
 239
 240         if (slave->nb_inflight_cops) {
 241                 nb_deq_ops_sec = rte_cryptodev_dequeue_burst(slave->dev_id,
 242                                 slave->qp_id, &ops[nb_deq_ops_pri],
 243                                 nb_ops - nb_deq_ops_pri);
 244                 slave->nb_inflight_cops -= nb_deq_ops_sec;
 245
 246                 if (!slave->nb_inflight_cops)
 247                         qp_ctx->deq_idx = (~qp_ctx->deq_idx) &
 248                                         SLAVE_IDX_SWITCH_MASK;
 249         }
 250
 251         return nb_deq_ops_pri + nb_deq_ops_sec;
 252 }
 253
 254 static uint16_t
 255 schedule_dequeue_ordering(void *qp, struct rte_crypto_op **ops,
 256                 uint16_t nb_ops)
 257 {
 258         struct rte_ring *order_ring =
 259                         ((struct scheduler_qp_ctx *)qp)->order_ring;
 260
 261         schedule_dequeue(qp, ops, nb_ops);
 262
 263         return scheduler_order_drain(order_ring, ops, nb_ops);
 264 }
 265
 266 static int
 267 slave_attach(__rte_unused struct rte_cryptodev *dev,
 268                 __rte_unused uint8_t slave_id)
 269 {
 270         return 0;
 271 }
 272
 273 static int
 274 slave_detach(__rte_unused struct rte_cryptodev *dev,
 275                 __rte_unused uint8_t slave_id)
 276 {
 277         return 0;
 278 }
 279
 280 static int
 281 scheduler_start(struct rte_cryptodev *dev)
 282 {
 283         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 284         struct psd_scheduler_ctx *psd_ctx = sched_ctx->private_ctx;
 285         uint16_t i;
 286
 287         /* for packet size based scheduler, nb_slaves have to >= 2 */
 288         if (sched_ctx->nb_slaves < NB_PKT_SIZE_SLAVES) {
 289                 CS_LOG_ERR("not enough slaves to start");
 290                 return -1;
 291         }
 292
 293         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
 294                 struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[i];
 295                 struct psd_scheduler_qp_ctx *ps_qp_ctx =
 296                                 qp_ctx->private_qp_ctx;
 297
 298                 ps_qp_ctx->primary_slave.dev_id =
 299                                 sched_ctx->slaves[PRIMARY_SLAVE_IDX].dev_id;
 300                 ps_qp_ctx->primary_slave.qp_id = i;
 301                 ps_qp_ctx->primary_slave.nb_inflight_cops = 0;
 302
 303                 ps_qp_ctx->secondary_slave.dev_id =
 304                                 sched_ctx->slaves[SECONDARY_SLAVE_IDX].dev_id;
 305                 ps_qp_ctx->secondary_slave.qp_id = i;
 306                 ps_qp_ctx->secondary_slave.nb_inflight_cops = 0;
 307
 308                 ps_qp_ctx->threshold = psd_ctx->threshold;
 309         }
 310
 311         if (sched_ctx->reordering_enabled) {
 312                 dev->enqueue_burst = &schedule_enqueue_ordering;
 313                 dev->dequeue_burst = &schedule_dequeue_ordering;
 314         } else {
 315                 dev->enqueue_burst = &schedule_enqueue;
 316                 dev->dequeue_burst = &schedule_dequeue;
 317         }
 318
 319         return 0;
 320 }
 321
 322 static int
 323 scheduler_stop(struct rte_cryptodev *dev)
 324 {
 325         uint16_t i;
 326
 327         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
 328                 struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[i];
 329                 struct psd_scheduler_qp_ctx *ps_qp_ctx = qp_ctx->private_qp_ctx;
 330
 331                 if (ps_qp_ctx->primary_slave.nb_inflight_cops +
 332                                 ps_qp_ctx->secondary_slave.nb_inflight_cops) {
 333                         CS_LOG_ERR("Some crypto ops left in slave queue");
 334                         return -1;
 335                 }
 336         }
 337
 338         return 0;
 339 }
 340
 341 static int
 342 scheduler_config_qp(struct rte_cryptodev *dev, uint16_t qp_id)
 343 {
 344         struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[qp_id];
 345         struct psd_scheduler_qp_ctx *ps_qp_ctx;
 346
 347         ps_qp_ctx = rte_zmalloc_socket(NULL, sizeof(*ps_qp_ctx), 0,
 348                         rte_socket_id());
 349         if (!ps_qp_ctx) {
 350                 CS_LOG_ERR("failed allocate memory for private queue pair");
 351                 return -ENOMEM;
 352         }
 353
 354         qp_ctx->private_qp_ctx = (void *)ps_qp_ctx;
 355
 356         return 0;
 357 }
 358
 359 static int
 360 scheduler_create_private_ctx(struct rte_cryptodev *dev)
 361 {
 362         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 363         struct psd_scheduler_ctx *psd_ctx;
 364
 365         if (sched_ctx->private_ctx)
 366                 rte_free(sched_ctx->private_ctx);
 367
 368         psd_ctx = rte_zmalloc_socket(NULL, sizeof(struct psd_scheduler_ctx), 0,
 369                         rte_socket_id());
 370         if (!psd_ctx) {
 371                 CS_LOG_ERR("failed allocate memory");
 372                 return -ENOMEM;
 373         }
 374
 375         psd_ctx->threshold = DEF_PKT_SIZE_THRESHOLD;
 376
 377         sched_ctx->private_ctx = (void *)psd_ctx;
 378
 379         return 0;
 380 }
 381 static int
 382 scheduler_option_set(struct rte_cryptodev *dev, uint32_t option_type,
 383                 void *option)
 384 {
 385         struct psd_scheduler_ctx *psd_ctx = ((struct scheduler_ctx *)
 386                         dev->data->dev_private)->private_ctx;
 387         uint32_t threshold;
 388
 389         if ((enum rte_cryptodev_schedule_option_type)option_type !=
 390                         CDEV_SCHED_OPTION_THRESHOLD) {
 391                 CS_LOG_ERR("Option not supported");
 392                 return -EINVAL;
 393         }
 394
 395         threshold = ((struct rte_cryptodev_scheduler_threshold_option *)
 396                         option)->threshold;
 397         if (!rte_is_power_of_2(threshold)) {
 398                 CS_LOG_ERR("Threshold is not power of 2");
 399                 return -EINVAL;
 400         }
 401
 402         psd_ctx->threshold = ~(threshold - 1);
 403
 404         return 0;
 405 }
 406
 407 static int
 408 scheduler_option_get(struct rte_cryptodev *dev, uint32_t option_type,
 409                 void *option)
 410 {
 411         struct psd_scheduler_ctx *psd_ctx = ((struct scheduler_ctx *)
 412                         dev->data->dev_private)->private_ctx;
 413         struct rte_cryptodev_scheduler_threshold_option *threshold_option;
 414
 415         if ((enum rte_cryptodev_schedule_option_type)option_type !=
 416                         CDEV_SCHED_OPTION_THRESHOLD) {
 417                 CS_LOG_ERR("Option not supported");
 418                 return -EINVAL;
 419         }
 420
 421         threshold_option = option;
 422         threshold_option->threshold = (~psd_ctx->threshold) + 1;
 423
 424         return 0;
 425 }
 426
 427 struct rte_cryptodev_scheduler_ops scheduler_ps_ops = {
 428         slave_attach,
 429         slave_detach,
 430         scheduler_start,
 431         scheduler_stop,
 432         scheduler_config_qp,
 433         scheduler_create_private_ctx,
 434         scheduler_option_set,
 435         scheduler_option_get
 436 };
 437
 438 struct rte_cryptodev_scheduler psd_scheduler = {
 439                 .name = "packet-size-based-scheduler",
 440                 .description = "scheduler which will distribute crypto op "
 441                                 "burst based on the packet size",
 442                 .mode = CDEV_SCHED_MODE_PKT_SIZE_DISTR,
 443                 .ops = &scheduler_ps_ops
 444 };
 445
 446 struct rte_cryptodev_scheduler *pkt_size_based_distr_scheduler = &psd_scheduler;