2 * Copyright (c) 2017 Intel and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vnet/vnet.h>
17 #include <dpdk/device/dpdk.h>
18 #include <dpdk/ipsec/ipsec.h>
21 format_crypto_resource (u8 * s, va_list * args)
23 dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
25 u32 indent = va_arg (*args, u32);
26 u32 res_idx = va_arg (*args, u32);
28 crypto_resource_t *res = vec_elt_at_index (dcm->resource, res_idx);
31 s = format (s, "%U thr_id %3d qp %2u dec_inflight %u, enc_inflights %u\n",
32 format_white_space, indent, (i16) res->thread_idx,
33 res->qp_id, res->inflights[0], res->inflights[1]);
39 format_crypto (u8 * s, va_list * args)
41 dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
42 crypto_dev_t *dev = va_arg (*args, crypto_dev_t *);
43 crypto_drv_t *drv = vec_elt_at_index (dcm->drv, dev->drv_id);
48 s = format (s, "%-25s%-20s%-10s\n", dev->name, drv->name,
49 rte_cryptodevs[dev->id].data->dev_started ? "up" : "down");
50 s = format (s, " numa_node %u, max_queues %u\n", dev->numa, dev->max_qp);
54 for (mask = 1; mask != 0; mask <<= 1)
56 feat = dev->features & mask;
60 format (s, "%s%s", pre,
61 rte_cryptodev_get_feature_name (feat));
68 s = format (s, " Cipher:");
70 for (i = 0; i < IPSEC_CRYPTO_N_ALG; i++)
71 if (dev->cipher_support[i])
73 s = format (s, "%s%s", pre, dcm->cipher_algs[i].name);
78 s = format (s, " Auth:");
80 for (i = 0; i < IPSEC_INTEG_N_ALG; i++)
81 if (dev->auth_support[i])
83 s = format (s, "%s%s", pre, dcm->auth_algs[i].name);
88 struct rte_cryptodev_stats stats;
89 rte_cryptodev_stats_get (dev->id, &stats);
93 " enqueue %-10lu dequeue %-10lu enqueue_err %-10lu dequeue_err %-10lu \n",
94 stats.enqueued_count, stats.dequeued_count,
95 stats.enqueue_err_count, stats.dequeue_err_count);
98 s = format (s, " free_resources %u :", vec_len (dev->free_resources));
100 u32 indent = format_get_indent (s);
101 s = format (s, "\n");
104 vec_foreach (res_idx, dev->free_resources)
105 s = format (s, "%U", format_crypto_resource, indent, res_idx[0]);
108 s = format (s, " used_resources %u :", vec_len (dev->used_resources));
109 indent = format_get_indent (s);
111 s = format (s, "\n");
114 vec_foreach (res_idx, dev->used_resources)
115 s = format (s, "%U", format_crypto_resource, indent, res_idx[0]);
118 s = format (s, "\n");
124 static clib_error_t *
125 clear_crypto_stats_fn (vlib_main_t * vm, unformat_input_t * input,
126 vlib_cli_command_t * cmd)
128 dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
132 vec_foreach (dev, dcm->dev)
133 rte_cryptodev_stats_reset (dev->id);
140 * This command is used to clear the DPDK Crypto device statistics.
143 * Example of how to clear the DPDK Crypto device statistics:
144 * @cliexstart{clear dpdk crypto devices statistics}
145 * vpp# clear dpdk crypto devices statistics
149 VLIB_CLI_COMMAND (clear_dpdk_crypto_stats, static) = {
150 .path = "clear dpdk crypto devices statistics",
151 .short_help = "clear dpdk crypto devices statistics",
152 .function = clear_crypto_stats_fn,
157 static clib_error_t *
158 show_dpdk_crypto_fn (vlib_main_t * vm, unformat_input_t * input,
159 vlib_cli_command_t * cmd)
161 dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
165 vec_foreach (dev, dcm->dev)
166 vlib_cli_output (vm, "%U", format_crypto, dev);
173 * This command is used to display the DPDK Crypto device information.
176 * Example of how to display the DPDK Crypto device information:
177 * @cliexstart{show dpdk crypto devices}
178 * vpp# show dpdk crypto devices
179 * aesni_mb0 crypto_aesni_mb up
180 * numa_node 0, max_queues 4
181 * SYMMETRIC_CRYPTO, SYM_OPERATION_CHAINING, CPU_AVX2, CPU_AESNI
182 * Cipher: aes-cbc-128, aes-cbc-192, aes-cbc-256, aes-ctr-128, aes-ctr-192,
183 * aes-ctr-256, aes-gcm-128, aes-gcm-192, aes-gcm-256
184 * Auth: md5-96, sha1-96, sha-256-128, sha-384-192, sha-512-256
185 * enqueue 2 dequeue 2 enqueue_err 0 dequeue_err 0
187 * thr_id -1 qp 3 inflight 0
188 * thr_id -1 qp 2 inflight 0
189 * thr_id -1 qp 1 inflight 0
191 * thr_id 1 qp 0 inflight 0
193 * Example of displaying the DPDK Crypto device data when enabled:
197 VLIB_CLI_COMMAND (show_dpdk_crypto, static) = {
198 .path = "show dpdk crypto devices",
199 .short_help = "show dpdk crypto devices",
200 .function = show_dpdk_crypto_fn,
205 format_crypto_worker (u8 * s, va_list * args)
207 u32 thread_idx = va_arg (*args, u32);
208 u8 verbose = (u8) va_arg (*args, u32);
209 dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
210 crypto_worker_main_t *cwm;
211 crypto_resource_t *res;
216 cwm = vec_elt_at_index (dcm->workers_main, thread_idx);
218 s = format (s, "Thread %u (%v):\n", thread_idx,
219 vlib_worker_threads[thread_idx].name);
222 vec_foreach (res_idx, cwm->resource_idx)
225 res = vec_elt_at_index (dcm->resource, res_idx[0]);
226 s = format (s, "%s%-20s dev-id %2u queue-pair %2u\n",
227 ind, vec_elt_at_index (dcm->dev, res->dev_id)->name,
228 res->dev_id, res->qp_id);
233 s = format (s, "%sCipher:", ind);
235 for (i = 0; i < IPSEC_CRYPTO_N_ALG; i++)
236 if (cwm->cipher_resource_idx[i] == res_idx[0])
238 s = format (s, "%s%s", pre, dcm->cipher_algs[i].name);
241 s = format (s, "\n");
243 s = format (s, "%sAuth:", ind);
245 for (i = 0; i < IPSEC_INTEG_N_ALG; i++)
246 if (cwm->auth_resource_idx[i] == res_idx[0])
248 s = format (s, "%s%s", pre, dcm->auth_algs[i].name);
251 s = format (s, "\n");
259 static clib_error_t *
260 common_crypto_placement_fn (vlib_main_t * vm, unformat_input_t * input,
261 vlib_cli_command_t * cmd, u8 verbose)
263 dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
264 clib_error_t *error = NULL;
270 vlib_cli_output (vm, "\nDPDK Cryptodev support is disabled\n");
274 skip_master = vlib_num_workers () > 0;
277 vec_foreach_index (i, dcm->workers_main)
282 vlib_cli_output (vm, "%U\n", format_crypto_worker, i, verbose);
289 static clib_error_t *
290 show_dpdk_crypto_placement_fn (vlib_main_t * vm, unformat_input_t * input,
291 vlib_cli_command_t * cmd)
293 return common_crypto_placement_fn (vm, input, cmd, 0);
296 static clib_error_t *
297 show_dpdk_crypto_placement_v_fn (vlib_main_t * vm, unformat_input_t * input,
298 vlib_cli_command_t * cmd)
300 return common_crypto_placement_fn (vm, input, cmd, 1);
304 * This command is used to display the DPDK Crypto device placement.
307 * Example of displaying the DPDK Crypto device placement:
308 * @cliexstart{show dpdk crypto placement}
309 * vpp# show dpdk crypto placement
310 * Thread 1 (vpp_wk_0):
311 * cryptodev_aesni_mb_p dev-id 0 queue-pair 0
312 * cryptodev_aesni_gcm_ dev-id 1 queue-pair 0
314 * Thread 2 (vpp_wk_1):
315 * cryptodev_aesni_mb_p dev-id 0 queue-pair 1
316 * cryptodev_aesni_gcm_ dev-id 1 queue-pair 1
320 VLIB_CLI_COMMAND (show_dpdk_crypto_placement, static) = {
321 .path = "show dpdk crypto placement",
322 .short_help = "show dpdk crypto placement",
323 .function = show_dpdk_crypto_placement_fn,
328 * This command is used to display the DPDK Crypto device placement
329 * with verbose output.
332 * Example of displaying the DPDK Crypto device placement verbose:
333 * @cliexstart{show dpdk crypto placement verbose}
334 * vpp# show dpdk crypto placement verbose
335 * Thread 1 (vpp_wk_0):
336 * cryptodev_aesni_mb_p dev-id 0 queue-pair 0
337 * Cipher: aes-cbc-128, aes-cbc-192, aes-cbc-256, aes-ctr-128, aes-ctr-192, aes-ctr-256
338 * Auth: md5-96, sha1-96, sha-256-128, sha-384-192, sha-512-256
339 * cryptodev_aesni_gcm_ dev-id 1 queue-pair 0
340 * Cipher: aes-gcm-128, aes-gcm-192, aes-gcm-256
343 * Thread 2 (vpp_wk_1):
344 * cryptodev_aesni_mb_p dev-id 0 queue-pair 1
345 * Cipher: aes-cbc-128, aes-cbc-192, aes-cbc-256, aes-ctr-128, aes-ctr-192, aes-ctr-256
346 * Auth: md5-96, sha1-96, sha-256-128, sha-384-192, sha-512-256
347 * cryptodev_aesni_gcm_ dev-id 1 queue-pair 1
348 * Cipher: aes-gcm-128, aes-gcm-192, aes-gcm-256
354 VLIB_CLI_COMMAND (show_dpdk_crypto_placement_v, static) = {
355 .path = "show dpdk crypto placement verbose",
356 .short_help = "show dpdk crypto placement verbose",
357 .function = show_dpdk_crypto_placement_v_fn,
361 static clib_error_t *
362 set_dpdk_crypto_placement_fn (vlib_main_t * vm,
363 unformat_input_t * input,
364 vlib_cli_command_t * cmd)
366 unformat_input_t _line_input, *line_input = &_line_input;
367 dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
368 crypto_worker_main_t *cwm;
372 u8 dev_idx, auto_en = 0;
374 if (!unformat_user (input, unformat_line_input, line_input))
375 return clib_error_return (0, "invalid syntax");
377 while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
379 if (unformat (line_input, "%u %u", &dev_idx, &thread_idx))
381 else if (unformat (line_input, "auto"))
385 unformat_free (line_input);
386 return clib_error_return (0, "parse error: '%U'",
387 format_unformat_error, line_input);
391 unformat_free (line_input);
395 crypto_auto_placement ();
399 /* TODO support device name */
401 if (!(dev_idx < vec_len (dcm->dev)))
402 return clib_error_return (0, "please specify valid device index");
404 if (thread_idx != (u32) ~ 0 && !(thread_idx < vec_len (dcm->workers_main)))
405 return clib_error_return (0, "invalid thread index");
407 dev = vec_elt_at_index (dcm->dev, dev_idx);
408 if (!(vec_len (dev->free_resources)))
409 return clib_error_return (0, "all device resources are being used");
411 /* Check thread is not already using the device */
413 vec_foreach (idx, dev->used_resources)
414 if (dcm->resource[idx[0]].thread_idx == thread_idx)
415 return clib_error_return (0, "thread %u already using device %u",
416 thread_idx, dev_idx);
419 res_idx = vec_pop (dev->free_resources);
420 vec_add1 (dev->used_resources, res_idx);
422 cwm = vec_elt_at_index (dcm->workers_main, thread_idx);
424 ASSERT (dcm->resource[res_idx].thread_idx == (u16) ~ 0);
425 dcm->resource[res_idx].thread_idx = thread_idx;
427 /* Add device to vector of polling resources */
428 vec_add1 (cwm->resource_idx, res_idx);
430 /* Set device as default for all supported algos */
431 for (i = 0; i < IPSEC_CRYPTO_N_ALG; i++)
432 if (dev->cipher_support[i])
434 if (cwm->cipher_resource_idx[i] == (u16) ~ 0)
435 dcm->cipher_algs[i].disabled--;
436 cwm->cipher_resource_idx[i] = res_idx;
439 for (i = 0; i < IPSEC_INTEG_N_ALG; i++)
440 if (dev->auth_support[i])
442 if (cwm->auth_resource_idx[i] == (u16) ~ 0)
443 dcm->auth_algs[i].disabled--;
444 cwm->auth_resource_idx[i] = res_idx;
447 /* Check if any unused resource */
451 vec_foreach (idx, cwm->resource_idx)
453 if (idx[0] == res_idx)
456 for (i = 0; i < IPSEC_CRYPTO_N_ALG; i++)
457 used |= cwm->cipher_resource_idx[i] == idx[0];
459 for (i = 0; i < IPSEC_INTEG_N_ALG; i++)
460 used |= cwm->auth_resource_idx[i] == idx[0];
462 vec_elt_at_index (dcm->resource, idx[0])->remove = !used;
470 VLIB_CLI_COMMAND (set_dpdk_crypto_placement, static) = {
471 .path = "set dpdk crypto placement",
472 .short_help = "set dpdk crypto placement (<device> <thread> | auto)",
473 .function = set_dpdk_crypto_placement_fn,
478 * The thread will not enqueue more operations to the device but will poll
479 * from it until there are no more inflight operations.
482 dpdk_crypto_clear_resource (u16 res_idx)
484 dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
485 crypto_resource_t *res = vec_elt_at_index (dcm->resource, res_idx);
486 crypto_worker_main_t *cwm = &dcm->workers_main[res->thread_idx];
489 for (i = 0; i < IPSEC_CRYPTO_N_ALG; i++)
490 if (cwm->cipher_resource_idx[i] == res_idx)
492 cwm->cipher_resource_idx[i] = (u16) ~ 0;
493 dcm->cipher_algs[i].disabled++;
496 for (i = 0; i < IPSEC_INTEG_N_ALG; i++)
497 if (cwm->auth_resource_idx[i] == res_idx)
499 cwm->auth_resource_idx[i] = (u16) ~ 0;
500 dcm->auth_algs[i].disabled++;
503 /* Fully remove device on crypto_node once there are no inflights */
507 static clib_error_t *
508 clear_dpdk_crypto_placement_fn (vlib_main_t * vm,
510 input, vlib_cli_command_t * cmd)
512 unformat_input_t _line_input, *line_input = &_line_input;
513 dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
515 u32 thread_idx = (u32) ~ 0;
517 u8 dev_idx = (u8) ~ 0;
520 if (!unformat_user (input, unformat_line_input, line_input))
521 return clib_error_return (0, "invalid syntax");
523 while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
525 if (unformat (line_input, "%u %u", &dev_idx, &thread_idx))
527 else if (unformat (line_input, "%u", &dev_idx))
531 unformat_free (line_input);
532 return clib_error_return (0, "parse error: '%U'",
533 format_unformat_error, line_input);
537 unformat_free (line_input);
539 if (!(dev_idx < vec_len (dcm->dev)))
540 return clib_error_return (0, "invalid device index");
542 dev = vec_elt_at_index (dcm->dev, dev_idx);
544 /* Clear all resources placements */
548 vec_foreach (res_idx, dev->used_resources)
549 dpdk_crypto_clear_resource (res_idx[0]);
555 if (!(thread_idx < vec_len (dcm->workers_main)))
556 return clib_error_return (0, "invalid thread index");
558 /* Clear placement of device for given thread index */
560 vec_foreach (res_idx, dev->used_resources)
561 if (dcm->resource[res_idx[0]].thread_idx == thread_idx)
565 if (!(res_idx < vec_end (dev->used_resources)))
566 return clib_error_return (0, "thread %u is not using device %u",
567 thread_idx, dev_idx);
569 dpdk_crypto_clear_resource (res_idx[0]);
575 VLIB_CLI_COMMAND (clear_dpdk_crypto_placement, static) = {
576 .path = "clear dpdk crypto placement",
577 .short_help = "clear dpdk crypto placement <device> [<thread>]",
578 .function = clear_dpdk_crypto_placement_fn,
583 format_dpdk_mempool (u8 * s, va_list * args)
585 struct rte_mempool *mp = va_arg (*args, struct rte_mempool *);
586 u32 indent = format_get_indent (s);
587 u32 count = rte_mempool_avail_count (mp);
589 s = format (s, "%s\n%Uavailable %7d, allocated %7d total %7d\n",
590 mp->name, format_white_space, indent + 2,
591 count, mp->size - count, mp->size);
592 s = format (s, "%Uphys_addr %p, flags %08x, nb_mem_chunks %u\n",
593 format_white_space, indent + 2,
594 mp->mz->iova, mp->flags, mp->nb_mem_chunks);
595 s = format (s, "%Uelt_size %4u, header_size %3u, trailer_size %u\n",
596 format_white_space, indent + 2,
597 mp->elt_size, mp->header_size, mp->trailer_size);
598 s = format (s, "%Uprivate_data_size %3u, total_elt_size %u\n",
599 format_white_space, indent + 2,
600 mp->private_data_size,
601 mp->elt_size + mp->header_size + mp->trailer_size);
605 static clib_error_t *
606 show_dpdk_crypto_pools_fn (vlib_main_t * vm,
607 unformat_input_t * input, vlib_cli_command_t * cmd)
609 dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
613 vec_foreach (data, dcm->data)
616 vlib_cli_output (vm, "%U\n", format_dpdk_mempool, data->crypto_op);
618 vlib_cli_output (vm, "%U\n", format_dpdk_mempool, data->session_h);
620 struct rte_mempool **mp;
621 vec_foreach (mp, data->session_drv)
623 vlib_cli_output (vm, "%U\n", format_dpdk_mempool, mp[0]);
631 * This command is used to display the DPDK Crypto pools information.
634 * Example of how to display the DPDK Crypto pools information:
635 * @cliexstart{show crypto device mapping}
636 * vpp# show dpdk crypto pools
638 * available 15872, allocated 512 total 16384
639 * phys_addr 0xf3d2086c0, flags 00000010, nb_mem_chunks 1
640 * elt_size 160, header_size 64, trailer_size 96
641 * private_data_size 64, total_elt_size 320
643 * session_h_pool_numa1
644 * available 19998, allocated 2 total 20000
645 * phys_addr 0xf3c9c4380, flags 00000010, nb_mem_chunks 1
646 * elt_size 40, header_size 64, trailer_size 88
647 * private_data_size 0, total_elt_size 192
649 * session_drv0_pool_numa1
650 * available 19998, allocated 2 total 20000
651 * phys_addr 0xf3ad42d80, flags 00000010, nb_mem_chunks 1
652 * elt_size 512, header_size 64, trailer_size 0
653 * private_data_size 0, total_elt_size 576
657 VLIB_CLI_COMMAND (show_dpdk_crypto_pools, static) = {
658 .path = "show dpdk crypto pools",
659 .short_help = "show dpdk crypto pools",
660 .function = show_dpdk_crypto_pools_fn,
664 /* TODO Allow user define number of sessions supported */
665 /* TODO Allow user define descriptor queue size */
668 * fd.io coding-style-patch-verification: ON
671 * eval: (c-set-style "gnu")