2 * Copyright (c) 2017 Intel and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vnet/vnet.h>
17 #include <dpdk/device/dpdk.h>
18 #include <dpdk/ipsec/ipsec.h>
21 format_crypto_resource (u8 * s, va_list * args)
23 dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
25 u32 indent = va_arg (*args, u32);
26 u32 res_idx = va_arg (*args, u32);
28 crypto_resource_t *res = vec_elt_at_index (dcm->resource, res_idx);
31 s = format (s, "%U thr_id %3d qp %2u enc_inflight %u, dec_inflights %u\n",
32 format_white_space, indent, (i16) res->thread_idx,
33 res->qp_id, res->inflights[0], res->inflights[1]);
39 format_crypto (u8 * s, va_list * args)
41 dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
42 crypto_dev_t *dev = va_arg (*args, crypto_dev_t *);
43 crypto_drv_t *drv = vec_elt_at_index (dcm->drv, dev->drv_id);
48 s = format (s, "%-25s%-20s%-10s\n", dev->name, drv->name,
49 rte_cryptodevs[dev->id].data->dev_started ? "up" : "down");
50 s = format (s, " numa_node %u, max_queues %u\n", dev->numa, dev->max_qp);
54 for (mask = 1; mask != 0; mask <<= 1)
56 feat = dev->features & mask;
60 format (s, "%s%s", pre,
61 rte_cryptodev_get_feature_name (feat));
68 s = format (s, " Cipher:");
70 for (i = 0; i < IPSEC_CRYPTO_N_ALG; i++)
71 if (dev->cipher_support[i])
73 s = format (s, "%s%s", pre, dcm->cipher_algs[i].name);
78 s = format (s, " Auth:");
80 for (i = 0; i < IPSEC_INTEG_N_ALG; i++)
81 if (dev->auth_support[i])
83 s = format (s, "%s%s", pre, dcm->auth_algs[i].name);
88 struct rte_cryptodev_stats stats;
89 rte_cryptodev_stats_get (dev->id, &stats);
93 " enqueue %-10lu dequeue %-10lu enqueue_err %-10lu dequeue_err %-10lu \n",
94 stats.enqueued_count, stats.dequeued_count,
95 stats.enqueue_err_count, stats.dequeue_err_count);
98 s = format (s, " free_resources %u :", vec_len (dev->free_resources));
100 u32 indent = format_get_indent (s);
101 s = format (s, "\n");
104 vec_foreach (res_idx, dev->free_resources)
105 s = format (s, "%U", format_crypto_resource, indent, res_idx[0]);
108 s = format (s, " used_resources %u :", vec_len (dev->used_resources));
109 indent = format_get_indent (s);
111 s = format (s, "\n");
114 vec_foreach (res_idx, dev->used_resources)
115 s = format (s, "%U", format_crypto_resource, indent, res_idx[0]);
118 s = format (s, "\n");
124 static clib_error_t *
125 clear_crypto_stats_fn (vlib_main_t * vm, unformat_input_t * input,
126 vlib_cli_command_t * cmd)
128 dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
132 vec_foreach (dev, dcm->dev)
133 rte_cryptodev_stats_reset (dev->id);
140 * This command is used to clear the DPDK Crypto device statistics.
143 * Example of how to clear the DPDK Crypto device statistics:
144 * @cliexsart{clear dpdk crypto devices statistics}
145 * vpp# clear dpdk crypto devices statistics
147 * Example of clearing the DPDK Crypto device statistic data:
151 VLIB_CLI_COMMAND (clear_dpdk_crypto_stats, static) = {
152 .path = "clear dpdk crypto devices statistics",
153 .short_help = "clear dpdk crypto devices statistics",
154 .function = clear_crypto_stats_fn,
159 static clib_error_t *
160 show_dpdk_crypto_fn (vlib_main_t * vm, unformat_input_t * input,
161 vlib_cli_command_t * cmd)
163 dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
167 vec_foreach (dev, dcm->dev)
168 vlib_cli_output (vm, "%U", format_crypto, dev);
175 * This command is used to display the DPDK Crypto device information.
178 * Example of how to display the DPDK Crypto device information:
179 * @cliexsart{show dpdk crypto devices}
180 * vpp# show dpdk crypto devices
181 * aesni_mb0 crypto_aesni_mb up
182 * numa_node 0, max_queues 4
183 * SYMMETRIC_CRYPTO, SYM_OPERATION_CHAINING, CPU_AVX2, CPU_AESNI
184 * Cipher: aes-cbc-128, aes-cbc-192, aes-cbc-256, aes-ctr-128, aes-ctr-192, aes-ctr-256, aes-gcm-128, aes-gcm-192, aes-gcm-256
185 * Auth: md5-96, sha1-96, sha-256-128, sha-384-192, sha-512-256
186 * enqueue 2 dequeue 2 enqueue_err 0 dequeue_err 0
188 * thr_id -1 qp 3 inflight 0
189 * thr_id -1 qp 2 inflight 0
190 * thr_id -1 qp 1 inflight 0
192 * thr_id 1 qp 0 inflight 0
194 * Example of displaying the DPDK Crypto device data when enabled:
198 VLIB_CLI_COMMAND (show_dpdk_crypto, static) = {
199 .path = "show dpdk crypto devices",
200 .short_help = "show dpdk crypto devices",
201 .function = show_dpdk_crypto_fn,
206 format_crypto_worker (u8 * s, va_list * args)
208 u32 thread_idx = va_arg (*args, u32);
209 u8 verbose = (u8) va_arg (*args, u32);
210 dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
211 crypto_worker_main_t *cwm;
212 crypto_resource_t *res;
217 cwm = vec_elt_at_index (dcm->workers_main, thread_idx);
219 s = format (s, "Thread %u (%v):\n", thread_idx,
220 vlib_worker_threads[thread_idx].name);
223 vec_foreach (res_idx, cwm->resource_idx)
226 res = vec_elt_at_index (dcm->resource, res_idx[0]);
227 s = format (s, "%s%-20s dev-id %2u queue-pair %2u\n",
228 ind, vec_elt_at_index (dcm->dev, res->dev_id)->name,
229 res->dev_id, res->qp_id);
234 s = format (s, "%sCipher:", ind);
236 for (i = 0; i < IPSEC_CRYPTO_N_ALG; i++)
237 if (cwm->cipher_resource_idx[i] == res_idx[0])
239 s = format (s, "%s%s", pre, dcm->cipher_algs[i].name);
242 s = format (s, "\n");
244 s = format (s, "%sAuth:", ind);
246 for (i = 0; i < IPSEC_INTEG_N_ALG; i++)
247 if (cwm->auth_resource_idx[i] == res_idx[0])
249 s = format (s, "%s%s", pre, dcm->auth_algs[i].name);
252 s = format (s, "\n");
260 static clib_error_t *
261 common_crypto_placement_fn (vlib_main_t * vm, unformat_input_t * input,
262 vlib_cli_command_t * cmd, u8 verbose)
264 dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
265 clib_error_t *error = NULL;
271 vlib_cli_output (vm, "\nDPDK Cryptodev support is disabled\n");
275 skip_master = vlib_num_workers () > 0;
278 vec_foreach_index (i, dcm->workers_main)
283 vlib_cli_output (vm, "%U\n", format_crypto_worker, i, verbose);
290 static clib_error_t *
291 show_dpdk_crypto_placement_fn (vlib_main_t * vm, unformat_input_t * input,
292 vlib_cli_command_t * cmd)
294 return common_crypto_placement_fn (vm, input, cmd, 0);
297 static clib_error_t *
298 show_dpdk_crypto_placement_v_fn (vlib_main_t * vm, unformat_input_t * input,
299 vlib_cli_command_t * cmd)
301 return common_crypto_placement_fn (vm, input, cmd, 1);
305 * This command is used to display the DPDK Crypto device placement.
308 * Example of displaying the DPDK Crypto device placement:
309 * @cliexstart{show dpdk crypto placement}
310 * vpp# show dpdk crypto placement
311 * Thread 1 (vpp_wk_0):
312 * cryptodev_aesni_mb_p dev-id 0 queue-pair 0
313 * cryptodev_aesni_gcm_ dev-id 1 queue-pair 0
315 * Thread 2 (vpp_wk_1):
316 * cryptodev_aesni_mb_p dev-id 0 queue-pair 1
317 * cryptodev_aesni_gcm_ dev-id 1 queue-pair 1
321 VLIB_CLI_COMMAND (show_dpdk_crypto_placement, static) = {
322 .path = "show dpdk crypto placement",
323 .short_help = "show dpdk crypto placement",
324 .function = show_dpdk_crypto_placement_fn,
329 * This command is used to display the DPDK Crypto device placement
330 * with verbose output.
333 * Example of displaying the DPDK Crypto device placement verbose:
334 * @cliexstart{show dpdk crypto placement verbose}
335 * vpp# show dpdk crypto placement verbose
336 * Thread 1 (vpp_wk_0):
337 * cryptodev_aesni_mb_p dev-id 0 queue-pair 0
338 * Cipher: aes-cbc-128, aes-cbc-192, aes-cbc-256, aes-ctr-128, aes-ctr-192, aes-ctr-256
339 * Auth: md5-96, sha1-96, sha-256-128, sha-384-192, sha-512-256
340 * cryptodev_aesni_gcm_ dev-id 1 queue-pair 0
341 * Cipher: aes-gcm-128, aes-gcm-192, aes-gcm-256
344 * Thread 2 (vpp_wk_1):
345 * cryptodev_aesni_mb_p dev-id 0 queue-pair 1
346 * Cipher: aes-cbc-128, aes-cbc-192, aes-cbc-256, aes-ctr-128, aes-ctr-192, aes-ctr-256
347 * Auth: md5-96, sha1-96, sha-256-128, sha-384-192, sha-512-256
348 * cryptodev_aesni_gcm_ dev-id 1 queue-pair 1
349 * Cipher: aes-gcm-128, aes-gcm-192, aes-gcm-256
355 VLIB_CLI_COMMAND (show_dpdk_crypto_placement_v, static) = {
356 .path = "show dpdk crypto placement verbose",
357 .short_help = "show dpdk crypto placement verbose",
358 .function = show_dpdk_crypto_placement_v_fn,
362 static clib_error_t *
363 set_dpdk_crypto_placement_fn (vlib_main_t * vm,
364 unformat_input_t * input,
365 vlib_cli_command_t * cmd)
367 unformat_input_t _line_input, *line_input = &_line_input;
368 dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
369 crypto_worker_main_t *cwm;
373 u8 dev_idx, auto_en = 0;
375 if (!unformat_user (input, unformat_line_input, line_input))
376 return clib_error_return (0, "invalid syntax");
378 while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
380 if (unformat (line_input, "%u %u", &dev_idx, &thread_idx))
382 else if (unformat (line_input, "auto"))
386 unformat_free (line_input);
387 return clib_error_return (0, "parse error: '%U'",
388 format_unformat_error, line_input);
392 unformat_free (line_input);
396 crypto_auto_placement ();
400 /* TODO support device name */
402 if (!(dev_idx < vec_len (dcm->dev)))
403 return clib_error_return (0, "please specify valid device index");
405 if (thread_idx != (u32) ~ 0 && !(thread_idx < vec_len (dcm->workers_main)))
406 return clib_error_return (0, "invalid thread index");
408 dev = vec_elt_at_index (dcm->dev, dev_idx);
409 if (!(vec_len (dev->free_resources)))
410 return clib_error_return (0, "all device resources are being used");
412 /* Check thread is not already using the device */
414 vec_foreach (idx, dev->used_resources)
415 if (dcm->resource[idx[0]].thread_idx == thread_idx)
416 return clib_error_return (0, "thread %u already using device %u",
417 thread_idx, dev_idx);
420 res_idx = vec_pop (dev->free_resources);
421 vec_add1 (dev->used_resources, res_idx);
423 cwm = vec_elt_at_index (dcm->workers_main, thread_idx);
425 ASSERT (dcm->resource[res_idx].thread_idx == (u16) ~ 0);
426 dcm->resource[res_idx].thread_idx = thread_idx;
428 /* Add device to vector of polling resources */
429 vec_add1 (cwm->resource_idx, res_idx);
431 /* Set device as default for all supported algos */
432 for (i = 0; i < IPSEC_CRYPTO_N_ALG; i++)
433 if (dev->cipher_support[i])
435 if (cwm->cipher_resource_idx[i] == (u16) ~ 0)
436 dcm->cipher_algs[i].disabled--;
437 cwm->cipher_resource_idx[i] = res_idx;
440 for (i = 0; i < IPSEC_INTEG_N_ALG; i++)
441 if (dev->auth_support[i])
443 if (cwm->auth_resource_idx[i] == (u16) ~ 0)
444 dcm->auth_algs[i].disabled--;
445 cwm->auth_resource_idx[i] = res_idx;
448 /* Check if any unused resource */
452 vec_foreach (idx, cwm->resource_idx)
454 if (idx[0] == res_idx)
457 for (i = 0; i < IPSEC_CRYPTO_N_ALG; i++)
458 used |= cwm->cipher_resource_idx[i] == idx[0];
460 for (i = 0; i < IPSEC_INTEG_N_ALG; i++)
461 used |= cwm->auth_resource_idx[i] == idx[0];
463 vec_elt_at_index (dcm->resource, idx[0])->remove = !used;
471 VLIB_CLI_COMMAND (set_dpdk_crypto_placement, static) = {
472 .path = "set dpdk crypto placement",
473 .short_help = "set dpdk crypto placement (<device> <thread> | auto)",
474 .function = set_dpdk_crypto_placement_fn,
479 * The thread will not enqueue more operations to the device but will poll
480 * from it until there are no more inflight operations.
483 dpdk_crypto_clear_resource (u16 res_idx)
485 dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
486 crypto_resource_t *res = vec_elt_at_index (dcm->resource, res_idx);
487 crypto_worker_main_t *cwm = &dcm->workers_main[res->thread_idx];
490 for (i = 0; i < IPSEC_CRYPTO_N_ALG; i++)
491 if (cwm->cipher_resource_idx[i] == res_idx)
493 cwm->cipher_resource_idx[i] = (u16) ~ 0;
494 dcm->cipher_algs[i].disabled++;
497 for (i = 0; i < IPSEC_INTEG_N_ALG; i++)
498 if (cwm->auth_resource_idx[i] == res_idx)
500 cwm->auth_resource_idx[i] = (u16) ~ 0;
501 dcm->auth_algs[i].disabled++;
504 /* Fully remove device on crypto_node once there are no inflights */
508 static clib_error_t *
509 clear_dpdk_crypto_placement_fn (vlib_main_t * vm,
511 input, vlib_cli_command_t * cmd)
513 unformat_input_t _line_input, *line_input = &_line_input;
514 dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
516 u32 thread_idx = (u32) ~ 0;
518 u8 dev_idx = (u8) ~ 0;
521 if (!unformat_user (input, unformat_line_input, line_input))
522 return clib_error_return (0, "invalid syntax");
524 while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
526 if (unformat (line_input, "%u %u", &dev_idx, &thread_idx))
528 else if (unformat (line_input, "%u", &dev_idx))
532 unformat_free (line_input);
533 return clib_error_return (0, "parse error: '%U'",
534 format_unformat_error, line_input);
538 unformat_free (line_input);
540 if (!(dev_idx < vec_len (dcm->dev)))
541 return clib_error_return (0, "invalid device index");
543 dev = vec_elt_at_index (dcm->dev, dev_idx);
545 /* Clear all resources placements */
549 vec_foreach (res_idx, dev->used_resources)
550 dpdk_crypto_clear_resource (res_idx[0]);
556 if (!(thread_idx < vec_len (dcm->workers_main)))
557 return clib_error_return (0, "invalid thread index");
559 /* Clear placement of device for given thread index */
561 vec_foreach (res_idx, dev->used_resources)
562 if (dcm->resource[res_idx[0]].thread_idx == thread_idx)
566 if (!(res_idx < vec_end (dev->used_resources)))
567 return clib_error_return (0, "thread %u is not using device %u",
568 thread_idx, dev_idx);
570 dpdk_crypto_clear_resource (res_idx[0]);
576 VLIB_CLI_COMMAND (clear_dpdk_crypto_placement, static) = {
577 .path = "clear dpdk crypto placement",
578 .short_help = "clear dpdk crypto placement <device> [<thread>]",
579 .function = clear_dpdk_crypto_placement_fn,
584 format_dpdk_mempool (u8 * s, va_list * args)
586 struct rte_mempool *mp = va_arg (*args, struct rte_mempool *);
587 u32 indent = format_get_indent (s);
588 u32 count = rte_mempool_avail_count (mp);
590 s = format (s, "%s\n%Uavailable %7d, allocated %7d total %7d\n",
591 mp->name, format_white_space, indent + 2,
592 count, mp->size - count, mp->size);
593 s = format (s, "%Uphys_addr %p, flags %08x, nb_mem_chunks %u\n",
594 format_white_space, indent + 2,
595 mp->mz->phys_addr, mp->flags, mp->nb_mem_chunks);
596 s = format (s, "%Uelt_size %4u, header_size %3u, trailer_size %u\n",
597 format_white_space, indent + 2,
598 mp->elt_size, mp->header_size, mp->trailer_size);
599 s = format (s, "%Uprivate_data_size %3u, total_elt_size %u\n",
600 format_white_space, indent + 2,
601 mp->private_data_size,
602 mp->elt_size + mp->header_size + mp->trailer_size);
606 static clib_error_t *
607 show_dpdk_crypto_pools_fn (vlib_main_t * vm,
608 unformat_input_t * input, vlib_cli_command_t * cmd)
610 dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
614 vec_foreach (data, dcm->data)
617 vlib_cli_output (vm, "%U\n", format_dpdk_mempool, data->crypto_op);
619 vlib_cli_output (vm, "%U\n", format_dpdk_mempool, data->session_h);
621 struct rte_mempool **mp;
622 vec_foreach (mp, data->session_drv)
624 vlib_cli_output (vm, "%U\n", format_dpdk_mempool, mp[0]);
632 * This command is used to display the DPDK Crypto pools information.
635 * Example of how to display the DPDK Crypto pools information:
636 * @cliexstart{show crypto device mapping}
637 * vpp# show dpdk crypto pools
639 * available 15872, allocated 512 total 16384
640 * phys_addr 0xf3d2086c0, flags 00000010, nb_mem_chunks 1
641 * elt_size 160, header_size 64, trailer_size 96
642 * private_data_size 64, total_elt_size 320
644 * session_h_pool_numa1
645 * available 19998, allocated 2 total 20000
646 * phys_addr 0xf3c9c4380, flags 00000010, nb_mem_chunks 1
647 * elt_size 40, header_size 64, trailer_size 88
648 * private_data_size 0, total_elt_size 192
650 * session_drv0_pool_numa1
651 * available 19998, allocated 2 total 20000
652 * phys_addr 0xf3ad42d80, flags 00000010, nb_mem_chunks 1
653 * elt_size 512, header_size 64, trailer_size 0
654 * private_data_size 0, total_elt_size 576
658 VLIB_CLI_COMMAND (show_dpdk_crypto_pools, static) = {
659 .path = "show dpdk crypto pools",
660 .short_help = "show dpdk crypto pools",
661 .function = show_dpdk_crypto_pools_fn,
665 /* TODO Allow user define number of sessions supported */
666 /* TODO Allow user define descriptor queue size */
669 * fd.io coding-style-patch-verification: ON
672 * eval: (c-set-style "gnu")