4 * Copyright(c) 2017 Intel Corporation. All rights reserved.
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
10 * * Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * * Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in
14 * the documentation and/or other materials provided with the
16 * * Neither the name of Intel Corporation nor the names of its
17 * contributors may be used to endorse or promote products derived
18 * from this software without specific prior written permission.
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 #include <rte_reorder.h>
33 #include <rte_cryptodev.h>
34 #include <rte_cryptodev_pmd.h>
35 #include <rte_malloc.h>
37 #include "rte_cryptodev_scheduler.h"
38 #include "scheduler_pmd_private.h"
40 /** update the scheduler pmd's capability with attaching device's
42 * For each device to be attached, the scheduler's capability should be
43 * the common capability set of all slaves
46 sync_caps(struct rte_cryptodev_capabilities *caps,
48 const struct rte_cryptodev_capabilities *slave_caps)
50 uint32_t sync_nb_caps = nb_caps, nb_slave_caps = 0;
53 while (slave_caps[nb_slave_caps].op != RTE_CRYPTO_OP_TYPE_UNDEFINED)
57 rte_memcpy(caps, slave_caps, sizeof(*caps) * nb_slave_caps);
61 for (i = 0; i < sync_nb_caps; i++) {
62 struct rte_cryptodev_capabilities *cap = &caps[i];
65 for (j = 0; j < nb_slave_caps; j++) {
66 const struct rte_cryptodev_capabilities *s_cap =
69 if (s_cap->op != cap->op || s_cap->sym.xform_type !=
73 if (s_cap->sym.xform_type ==
74 RTE_CRYPTO_SYM_XFORM_AUTH) {
75 if (s_cap->sym.auth.algo !=
79 cap->sym.auth.digest_size.min =
80 s_cap->sym.auth.digest_size.min <
81 cap->sym.auth.digest_size.min ?
82 s_cap->sym.auth.digest_size.min :
83 cap->sym.auth.digest_size.min;
84 cap->sym.auth.digest_size.max =
85 s_cap->sym.auth.digest_size.max <
86 cap->sym.auth.digest_size.max ?
87 s_cap->sym.auth.digest_size.max :
88 cap->sym.auth.digest_size.max;
92 if (s_cap->sym.xform_type ==
93 RTE_CRYPTO_SYM_XFORM_CIPHER)
94 if (s_cap->sym.cipher.algo !=
98 /* no common cap found */
102 if (j < nb_slave_caps)
105 /* remove a uncommon cap from the array */
106 for (j = i; j < sync_nb_caps - 1; j++)
107 rte_memcpy(&caps[j], &caps[j+1], sizeof(*cap));
109 memset(&caps[sync_nb_caps - 1], 0, sizeof(*cap));
117 update_scheduler_capability(struct scheduler_ctx *sched_ctx)
119 struct rte_cryptodev_capabilities tmp_caps[256] = { {0} };
120 uint32_t nb_caps = 0, i;
122 if (sched_ctx->capabilities) {
123 rte_free(sched_ctx->capabilities);
124 sched_ctx->capabilities = NULL;
127 for (i = 0; i < sched_ctx->nb_slaves; i++) {
128 struct rte_cryptodev_info dev_info;
130 rte_cryptodev_info_get(sched_ctx->slaves[i].dev_id, &dev_info);
132 nb_caps = sync_caps(tmp_caps, nb_caps, dev_info.capabilities);
137 sched_ctx->capabilities = rte_zmalloc_socket(NULL,
138 sizeof(struct rte_cryptodev_capabilities) *
139 (nb_caps + 1), 0, SOCKET_ID_ANY);
140 if (!sched_ctx->capabilities)
143 rte_memcpy(sched_ctx->capabilities, tmp_caps,
144 sizeof(struct rte_cryptodev_capabilities) * nb_caps);
150 update_scheduler_feature_flag(struct rte_cryptodev *dev)
152 struct scheduler_ctx *sched_ctx = dev->data->dev_private;
155 dev->feature_flags = 0;
157 for (i = 0; i < sched_ctx->nb_slaves; i++) {
158 struct rte_cryptodev_info dev_info;
160 rte_cryptodev_info_get(sched_ctx->slaves[i].dev_id, &dev_info);
162 dev->feature_flags |= dev_info.feature_flags;
167 update_max_nb_qp(struct scheduler_ctx *sched_ctx)
172 if (!sched_ctx->nb_slaves)
175 max_nb_qp = sched_ctx->nb_slaves ? UINT32_MAX : 0;
177 for (i = 0; i < sched_ctx->nb_slaves; i++) {
178 struct rte_cryptodev_info dev_info;
180 rte_cryptodev_info_get(sched_ctx->slaves[i].dev_id, &dev_info);
181 max_nb_qp = dev_info.max_nb_queue_pairs < max_nb_qp ?
182 dev_info.max_nb_queue_pairs : max_nb_qp;
185 sched_ctx->max_nb_queue_pairs = max_nb_qp;
188 /** Attach a device to the scheduler. */
190 rte_cryptodev_scheduler_slave_attach(uint8_t scheduler_id, uint8_t slave_id)
192 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
193 struct scheduler_ctx *sched_ctx;
194 struct scheduler_slave *slave;
195 struct rte_cryptodev_info dev_info;
199 CS_LOG_ERR("Operation not supported");
203 if (dev->driver_id != cryptodev_driver_id) {
204 CS_LOG_ERR("Operation not supported");
208 if (dev->data->dev_started) {
209 CS_LOG_ERR("Illegal operation");
213 sched_ctx = dev->data->dev_private;
214 if (sched_ctx->nb_slaves >=
215 RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES) {
216 CS_LOG_ERR("Too many slaves attached");
220 for (i = 0; i < sched_ctx->nb_slaves; i++)
221 if (sched_ctx->slaves[i].dev_id == slave_id) {
222 CS_LOG_ERR("Slave already added");
226 slave = &sched_ctx->slaves[sched_ctx->nb_slaves];
228 rte_cryptodev_info_get(slave_id, &dev_info);
230 slave->dev_id = slave_id;
231 slave->driver_id = dev_info.driver_id;
232 sched_ctx->nb_slaves++;
234 if (update_scheduler_capability(sched_ctx) < 0) {
236 slave->driver_id = 0;
237 sched_ctx->nb_slaves--;
239 CS_LOG_ERR("capabilities update failed");
243 update_scheduler_feature_flag(dev);
245 update_max_nb_qp(sched_ctx);
251 rte_cryptodev_scheduler_slave_detach(uint8_t scheduler_id, uint8_t slave_id)
253 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
254 struct scheduler_ctx *sched_ctx;
255 uint32_t i, slave_pos;
258 CS_LOG_ERR("Operation not supported");
262 if (dev->driver_id != cryptodev_driver_id) {
263 CS_LOG_ERR("Operation not supported");
267 if (dev->data->dev_started) {
268 CS_LOG_ERR("Illegal operation");
272 sched_ctx = dev->data->dev_private;
274 for (slave_pos = 0; slave_pos < sched_ctx->nb_slaves; slave_pos++)
275 if (sched_ctx->slaves[slave_pos].dev_id == slave_id)
277 if (slave_pos == sched_ctx->nb_slaves) {
278 CS_LOG_ERR("Cannot find slave");
282 if (sched_ctx->ops.slave_detach(dev, slave_id) < 0) {
283 CS_LOG_ERR("Failed to detach slave");
287 for (i = slave_pos; i < sched_ctx->nb_slaves - 1; i++) {
288 memcpy(&sched_ctx->slaves[i], &sched_ctx->slaves[i+1],
289 sizeof(struct scheduler_slave));
291 memset(&sched_ctx->slaves[sched_ctx->nb_slaves - 1], 0,
292 sizeof(struct scheduler_slave));
293 sched_ctx->nb_slaves--;
295 if (update_scheduler_capability(sched_ctx) < 0) {
296 CS_LOG_ERR("capabilities update failed");
300 update_scheduler_feature_flag(dev);
302 update_max_nb_qp(sched_ctx);
308 rte_cryptodev_scheduler_mode_set(uint8_t scheduler_id,
309 enum rte_cryptodev_scheduler_mode mode)
311 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
312 struct scheduler_ctx *sched_ctx;
315 CS_LOG_ERR("Operation not supported");
319 if (dev->driver_id != cryptodev_driver_id) {
320 CS_LOG_ERR("Operation not supported");
324 if (dev->data->dev_started) {
325 CS_LOG_ERR("Illegal operation");
329 sched_ctx = dev->data->dev_private;
331 if (mode == sched_ctx->mode)
335 case CDEV_SCHED_MODE_ROUNDROBIN:
336 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id,
337 roundrobin_scheduler) < 0) {
338 CS_LOG_ERR("Failed to load scheduler");
342 case CDEV_SCHED_MODE_PKT_SIZE_DISTR:
343 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id,
344 pkt_size_based_distr_scheduler) < 0) {
345 CS_LOG_ERR("Failed to load scheduler");
349 case CDEV_SCHED_MODE_FAILOVER:
350 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id,
351 failover_scheduler) < 0) {
352 CS_LOG_ERR("Failed to load scheduler");
356 case CDEV_SCHED_MODE_MULTICORE:
357 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id,
358 multicore_scheduler) < 0) {
359 CS_LOG_ERR("Failed to load scheduler");
364 CS_LOG_ERR("Not yet supported");
371 enum rte_cryptodev_scheduler_mode
372 rte_cryptodev_scheduler_mode_get(uint8_t scheduler_id)
374 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
375 struct scheduler_ctx *sched_ctx;
378 CS_LOG_ERR("Operation not supported");
382 if (dev->driver_id != cryptodev_driver_id) {
383 CS_LOG_ERR("Operation not supported");
387 sched_ctx = dev->data->dev_private;
389 return sched_ctx->mode;
393 rte_cryptodev_scheduler_ordering_set(uint8_t scheduler_id,
394 uint32_t enable_reorder)
396 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
397 struct scheduler_ctx *sched_ctx;
400 CS_LOG_ERR("Operation not supported");
404 if (dev->driver_id != cryptodev_driver_id) {
405 CS_LOG_ERR("Operation not supported");
409 if (dev->data->dev_started) {
410 CS_LOG_ERR("Illegal operation");
414 sched_ctx = dev->data->dev_private;
416 sched_ctx->reordering_enabled = enable_reorder;
422 rte_cryptodev_scheduler_ordering_get(uint8_t scheduler_id)
424 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
425 struct scheduler_ctx *sched_ctx;
428 CS_LOG_ERR("Operation not supported");
432 if (dev->driver_id != cryptodev_driver_id) {
433 CS_LOG_ERR("Operation not supported");
437 sched_ctx = dev->data->dev_private;
439 return (int)sched_ctx->reordering_enabled;
443 rte_cryptodev_scheduler_load_user_scheduler(uint8_t scheduler_id,
444 struct rte_cryptodev_scheduler *scheduler) {
446 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
447 struct scheduler_ctx *sched_ctx;
450 CS_LOG_ERR("Operation not supported");
454 if (dev->driver_id != cryptodev_driver_id) {
455 CS_LOG_ERR("Operation not supported");
459 if (dev->data->dev_started) {
460 CS_LOG_ERR("Illegal operation");
464 sched_ctx = dev->data->dev_private;
466 if (strlen(scheduler->name) > RTE_CRYPTODEV_NAME_MAX_LEN - 1) {
467 CS_LOG_ERR("Invalid name %s, should be less than "
468 "%u bytes.\n", scheduler->name,
469 RTE_CRYPTODEV_NAME_MAX_LEN);
472 snprintf(sched_ctx->name, sizeof(sched_ctx->name), "%s",
475 if (strlen(scheduler->description) >
476 RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN - 1) {
477 CS_LOG_ERR("Invalid description %s, should be less than "
478 "%u bytes.\n", scheduler->description,
479 RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN - 1);
482 snprintf(sched_ctx->description, sizeof(sched_ctx->description), "%s",
483 scheduler->description);
485 /* load scheduler instance operations functions */
486 sched_ctx->ops.config_queue_pair = scheduler->ops->config_queue_pair;
487 sched_ctx->ops.create_private_ctx = scheduler->ops->create_private_ctx;
488 sched_ctx->ops.scheduler_start = scheduler->ops->scheduler_start;
489 sched_ctx->ops.scheduler_stop = scheduler->ops->scheduler_stop;
490 sched_ctx->ops.slave_attach = scheduler->ops->slave_attach;
491 sched_ctx->ops.slave_detach = scheduler->ops->slave_detach;
492 sched_ctx->ops.option_set = scheduler->ops->option_set;
493 sched_ctx->ops.option_get = scheduler->ops->option_get;
495 if (sched_ctx->private_ctx) {
496 rte_free(sched_ctx->private_ctx);
497 sched_ctx->private_ctx = NULL;
500 if (sched_ctx->ops.create_private_ctx) {
501 int ret = (*sched_ctx->ops.create_private_ctx)(dev);
504 CS_LOG_ERR("Unable to create scheduler private "
510 sched_ctx->mode = scheduler->mode;
516 rte_cryptodev_scheduler_slaves_get(uint8_t scheduler_id, uint8_t *slaves)
518 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
519 struct scheduler_ctx *sched_ctx;
520 uint32_t nb_slaves = 0;
523 CS_LOG_ERR("Operation not supported");
527 if (dev->driver_id != cryptodev_driver_id) {
528 CS_LOG_ERR("Operation not supported");
532 sched_ctx = dev->data->dev_private;
534 nb_slaves = sched_ctx->nb_slaves;
536 if (slaves && nb_slaves) {
539 for (i = 0; i < nb_slaves; i++)
540 slaves[i] = sched_ctx->slaves[i].dev_id;
543 return (int)nb_slaves;
547 rte_cryptodev_scheduler_option_set(uint8_t scheduler_id,
548 enum rte_cryptodev_schedule_option_type option_type,
551 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
552 struct scheduler_ctx *sched_ctx;
554 if (option_type == CDEV_SCHED_OPTION_NOT_SET ||
555 option_type >= CDEV_SCHED_OPTION_COUNT) {
556 CS_LOG_ERR("Invalid option parameter");
561 CS_LOG_ERR("Invalid option parameter");
565 if (dev->data->dev_started) {
566 CS_LOG_ERR("Illegal operation");
570 sched_ctx = dev->data->dev_private;
572 RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx->ops.option_set, -ENOTSUP);
574 return (*sched_ctx->ops.option_set)(dev, option_type, option);
578 rte_cryptodev_scheduler_option_get(uint8_t scheduler_id,
579 enum rte_cryptodev_schedule_option_type option_type,
582 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
583 struct scheduler_ctx *sched_ctx;
586 CS_LOG_ERR("Operation not supported");
591 CS_LOG_ERR("Invalid option parameter");
595 if (dev->driver_id != cryptodev_driver_id) {
596 CS_LOG_ERR("Operation not supported");
600 sched_ctx = dev->data->dev_private;
602 RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx->ops.option_get, -ENOTSUP);
604 return (*sched_ctx->ops.option_get)(dev, option_type, option);