From 8d01b9cd70a67cdafd5b965a70420c3bd7fb3f82 Mon Sep 17 00:00:00 2001 From: Luca Boccassi Date: Thu, 1 Nov 2018 11:59:50 +0000 Subject: [PATCH] New upstream version 18.11-rc1 Change-Id: Iaa71986dd6332e878d8f4bf493101b2bbc6313bb Signed-off-by: Luca Boccassi --- MAINTAINERS | 71 +- app/meson.build | 11 +- app/pdump/main.c | 6 +- app/pdump/meson.build | 2 +- app/proc-info/meson.build | 2 +- app/test-bbdev/meson.build | 2 +- app/test-bbdev/test_bbdev_perf.c | 14 +- app/test-crypto-perf/cperf_test_vectors.c | 22 +- app/test-crypto-perf/main.c | 12 +- app/test-crypto-perf/meson.build | 2 +- app/test-eventdev/test_perf_common.c | 1 - app/test-eventdev/test_pipeline_atq.c | 287 +- app/test-eventdev/test_pipeline_common.c | 220 +- app/test-eventdev/test_pipeline_common.h | 70 +- app/test-eventdev/test_pipeline_queue.c | 260 +- app/test-pmd/Makefile | 4 +- app/test-pmd/cmdline.c | 984 +- app/test-pmd/cmdline_flow.c | 870 + app/test-pmd/cmdline_mtr.c | 6 +- app/test-pmd/cmdline_tm.c | 260 + app/test-pmd/cmdline_tm.h | 3 + app/test-pmd/config.c | 652 +- app/test-pmd/csumonly.c | 18 +- app/test-pmd/meson.build | 6 +- app/test-pmd/noisy_vnf.c | 279 + app/test-pmd/parameters.c | 107 +- app/test-pmd/rxonly.c | 138 +- app/test-pmd/testpmd.c | 779 +- app/test-pmd/testpmd.h | 129 +- app/test-pmd/util.c | 219 + buildtools/auto-config-h.sh | 4 +- buildtools/check-experimental-syms.sh | 4 +- buildtools/pmdinfogen/pmdinfogen.c | 1 - buildtools/symlink-drivers-solibs.sh | 2 +- config/arm/meson.build | 5 +- config/common_armv8a_linuxapp | 4 +- config/common_base | 53 +- config/common_linuxapp | 12 + config/defconfig_arm64-dpaa-linuxapp-gcc | 3 + config/defconfig_arm64-thunderx-linuxapp-gcc | 2 +- config/defconfig_ppc_64-power8-linuxapp-gcc | 1 + config/meson.build | 16 +- config/ppc_64/meson.build | 11 + config/rte_config.h | 12 +- devtools/check-forbidden-tokens.awk | 74 + devtools/check-git-log.sh | 3 + devtools/check-includes.sh | 4 +- devtools/check-symbol-change.sh | 8 +- devtools/checkpatches.sh | 92 +- devtools/cocci.sh | 2 +- devtools/test-build.sh | 8 +- devtools/test-meson-builds.sh | 9 +- doc/api/doxy-api-index.md | 3 + doc/api/doxy-api.conf | 87 - doc/api/doxy-api.conf.in | 97 + doc/api/generate_doxygen.sh | 10 + doc/api/generate_examples.sh | 12 + doc/api/meson.build | 55 + doc/build-sdk-meson.txt | 2 + doc/guides/compressdevs/features/octeontx.ini | 2 +- doc/guides/compressdevs/features/qat.ini | 1 + doc/guides/compressdevs/octeontx.rst | 24 +- doc/guides/compressdevs/qat_comp.rst | 11 +- doc/guides/contributing/coding_style.rst | 4 +- doc/guides/contributing/documentation.rst | 19 +- doc/guides/cryptodevs/aesni_mb.rst | 1 + doc/guides/cryptodevs/caam_jr.rst | 150 + doc/guides/cryptodevs/features/caam_jr.ini | 46 + doc/guides/cryptodevs/features/default.ini | 4 + doc/guides/cryptodevs/features/mvsam.ini | 12 + doc/guides/cryptodevs/features/octeontx.ini | 62 + doc/guides/cryptodevs/features/qat.ini | 4 + doc/guides/cryptodevs/index.rst | 2 + doc/guides/cryptodevs/mvsam.rst | 147 +- doc/guides/cryptodevs/octeontx.rst | 127 + doc/guides/cryptodevs/overview.rst | 2 +- doc/guides/cryptodevs/qat.rst | 200 +- doc/guides/eventdevs/dpaa.rst | 2 + doc/guides/eventdevs/dsw.rst | 96 + doc/guides/eventdevs/index.rst | 1 + doc/guides/eventdevs/octeontx.rst | 20 +- doc/guides/eventdevs/opdl.rst | 2 +- doc/guides/howto/index.rst | 1 + doc/guides/howto/rte_flow.rst | 32 +- doc/guides/howto/telemetry.rst | 85 + .../howto/virtio_user_as_exceptional_path.rst | 23 +- .../howto/virtio_user_for_container_networking.rst | 2 +- doc/guides/mempool/octeontx.rst | 18 +- doc/guides/meson.build | 28 + doc/guides/nics/atlantic.rst | 47 + doc/guides/nics/axgbe.rst | 2 +- doc/guides/nics/dpaa2.rst | 2 +- doc/guides/nics/ena.rst | 19 +- doc/guides/nics/enetc.rst | 110 + doc/guides/nics/enic.rst | 41 + doc/guides/nics/features.rst | 29 +- doc/guides/nics/features/atlantic.ini | 37 + doc/guides/nics/features/ena.ini | 1 + doc/guides/nics/features/enetc.ini | 11 + doc/guides/nics/features/failsafe.ini | 3 + doc/guides/nics/features/mvneta.ini | 19 + doc/guides/nics/features/netvsc.ini | 1 + doc/guides/nics/features/sfc_efx.ini | 2 + doc/guides/nics/fm10k.rst | 3 +- doc/guides/nics/i40e.rst | 15 + doc/guides/nics/img/mvpp2_tm.svg | 71 + doc/guides/nics/index.rst | 3 + doc/guides/nics/ixgbe.rst | 27 + doc/guides/nics/liquidio.rst | 8 - doc/guides/nics/mlx5.rst | 14 +- doc/guides/nics/mvneta.rst | 171 + doc/guides/nics/mvpp2.rst | 433 +- doc/guides/nics/netvsc.rst | 31 +- doc/guides/nics/octeontx.rst | 26 +- doc/guides/nics/pcap_ring.rst | 10 + doc/guides/nics/sfc_efx.rst | 9 +- doc/guides/nics/softnic.rst | 120 + doc/guides/nics/tap.rst | 16 + doc/guides/nics/vhost.rst | 5 + doc/guides/nics/virtio.rst | 2 +- doc/guides/platform/octeontx.rst | 16 +- doc/guides/prog_guide/env_abstraction_layer.rst | 45 + .../prog_guide/event_ethernet_tx_adapter.rst | 165 + doc/guides/prog_guide/hash_lib.rst | 33 +- doc/guides/prog_guide/index.rst | 2 +- doc/guides/prog_guide/kernel_nic_interface.rst | 239 +- doc/guides/prog_guide/packet_framework.rst | 11 +- doc/guides/prog_guide/port_hotplug_framework.rst | 106 - doc/guides/prog_guide/power_man.rst | 86 + doc/guides/prog_guide/profile_app.rst | 34 +- doc/guides/prog_guide/rte_flow.rst | 285 + doc/guides/prog_guide/rte_security.rst | 107 +- doc/guides/prog_guide/vhost_lib.rst | 8 + doc/guides/rel_notes/deprecation.rst | 40 - doc/guides/rel_notes/index.rst | 2 +- doc/guides/rel_notes/rel_description.rst | 12 - doc/guides/rel_notes/release_18_08.rst | 2 +- doc/guides/rel_notes/release_18_11.rst | 529 + doc/guides/sample_app_ug/flow_filtering.rst | 2 - doc/guides/sample_app_ug/index.rst | 1 + doc/guides/sample_app_ug/ip_pipeline.rst | 23 + doc/guides/sample_app_ug/ipsec_secgw.rst | 3 +- doc/guides/sample_app_ug/kernel_nic_interface.rst | 262 +- doc/guides/sample_app_ug/l3_forward_power_man.rst | 69 + doc/guides/sample_app_ug/link_status_intr.rst | 1 - doc/guides/sample_app_ug/vdpa.rst | 120 + doc/guides/sample_app_ug/vhost.rst | 2 +- doc/guides/sample_app_ug/vhost_crypto.rst | 26 +- doc/guides/sample_app_ug/vm_power_management.rst | 300 +- doc/guides/testpmd_app_ug/run_app.rst | 51 +- doc/guides/testpmd_app_ug/testpmd_funcs.rst | 445 +- .../img/eventdev_pipeline_atq_test_generic.svg | 848 +- ...> eventdev_pipeline_atq_test_internal_port.svg} | 26 +- .../img/eventdev_pipeline_queue_test_generic.svg | 570 +- ...eventdev_pipeline_queue_test_internal_port.svg} | 22 +- doc/guides/tools/testeventdev.rst | 44 +- doc/meson.build | 15 + drivers/Makefile | 1 + drivers/baseband/meson.build | 7 + drivers/baseband/null/meson.build | 7 + drivers/bus/dpaa/Makefile | 4 +- drivers/bus/dpaa/base/fman/netcfg_layer.c | 2 +- drivers/bus/dpaa/base/qbman/bman_driver.c | 17 +- drivers/bus/dpaa/base/qbman/qman.c | 72 +- drivers/bus/dpaa/base/qbman/qman_driver.c | 7 +- drivers/bus/dpaa/dpaa_bus.c | 19 +- drivers/bus/dpaa/include/compat.h | 20 +- drivers/bus/dpaa/include/fsl_fman_crc64.h | 8 +- drivers/bus/dpaa/include/fsl_qman.h | 20 + drivers/bus/dpaa/include/fsl_usd.h | 6 + drivers/bus/dpaa/meson.build | 5 +- drivers/bus/dpaa/rte_bus_dpaa_version.map | 16 +- drivers/bus/dpaa/rte_dpaa_bus.h | 6 + drivers/bus/fslmc/Makefile | 3 +- drivers/bus/fslmc/fslmc_bus.c | 35 +- drivers/bus/fslmc/fslmc_vfio.c | 13 +- drivers/bus/fslmc/mc/dpbp.c | 10 + drivers/bus/fslmc/mc/dpci.c | 197 + drivers/bus/fslmc/mc/dpcon.c | 30 + drivers/bus/fslmc/mc/dpdmai.c | 14 + drivers/bus/fslmc/mc/dpio.c | 9 + drivers/bus/fslmc/mc/fsl_dpbp.h | 1 + drivers/bus/fslmc/mc/fsl_dpbp_cmd.h | 16 +- drivers/bus/fslmc/mc/fsl_dpci.h | 47 +- drivers/bus/fslmc/mc/fsl_dpci_cmd.h | 62 +- drivers/bus/fslmc/mc/fsl_dpcon.h | 19 + drivers/bus/fslmc/mc/fsl_dpdmai.h | 5 + drivers/bus/fslmc/mc/fsl_dpdmai_cmd.h | 20 +- drivers/bus/fslmc/mc/fsl_dpmng.h | 2 +- drivers/bus/fslmc/mc/fsl_dpopr.h | 85 + drivers/bus/fslmc/meson.build | 5 +- drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c | 7 + drivers/bus/fslmc/portal/dpaa2_hw_dpio.c | 197 +- drivers/bus/fslmc/portal/dpaa2_hw_dpio.h | 4 + drivers/bus/fslmc/portal/dpaa2_hw_pvt.h | 53 +- drivers/bus/fslmc/qbman/include/compat.h | 3 +- drivers/bus/fslmc/qbman/include/fsl_qbman_portal.h | 33 +- drivers/bus/fslmc/qbman/qbman_portal.c | 764 +- drivers/bus/fslmc/qbman/qbman_portal.h | 30 +- drivers/bus/fslmc/qbman/qbman_sys.h | 100 +- drivers/bus/fslmc/qbman/qbman_sys_decl.h | 4 + drivers/bus/fslmc/rte_bus_fslmc_version.map | 13 + drivers/bus/ifpga/Makefile | 2 +- drivers/bus/ifpga/ifpga_bus.c | 33 +- drivers/bus/ifpga/meson.build | 2 + drivers/bus/ifpga/rte_bus_ifpga.h | 3 - drivers/bus/pci/Makefile | 5 +- drivers/bus/pci/bsd/pci.c | 6 +- drivers/bus/pci/linux/Makefile | 2 - drivers/bus/pci/linux/pci.c | 46 +- drivers/bus/pci/linux/pci_vfio.c | 268 +- drivers/bus/pci/meson.build | 9 +- drivers/bus/pci/pci_common.c | 143 +- drivers/bus/pci/pci_common_uio.c | 33 + drivers/bus/pci/pci_params.c | 78 + drivers/bus/pci/private.h | 39 + drivers/bus/pci/rte_bus_pci.h | 10 +- drivers/bus/vdev/Makefile | 5 +- drivers/bus/vdev/meson.build | 7 +- drivers/bus/vdev/vdev.c | 50 +- drivers/bus/vdev/vdev_params.c | 66 + drivers/bus/vdev/vdev_private.h | 26 + drivers/bus/vmbus/Makefile | 2 +- drivers/bus/vmbus/linux/vmbus_bus.c | 3 + drivers/bus/vmbus/meson.build | 2 + drivers/bus/vmbus/private.h | 6 + drivers/bus/vmbus/rte_bus_vmbus.h | 15 + drivers/bus/vmbus/rte_bus_vmbus_version.map | 7 + drivers/bus/vmbus/vmbus_channel.c | 26 + drivers/bus/vmbus/vmbus_common.c | 27 +- drivers/common/Makefile | 15 + drivers/common/cpt/Makefile | 25 + drivers/common/cpt/cpt_common.h | 91 + drivers/common/cpt/cpt_hw_types.h | 522 + drivers/common/cpt/cpt_mcode_defines.h | 386 + drivers/common/cpt/cpt_pmd_logs.h | 50 + drivers/common/cpt/cpt_pmd_ops_helper.c | 41 + drivers/common/cpt/cpt_pmd_ops_helper.h | 34 + drivers/common/cpt/cpt_request_mgr.h | 185 + drivers/common/cpt/cpt_ucode.h | 3648 +++ drivers/common/cpt/meson.build | 8 + drivers/common/cpt/rte_common_cpt_version.map | 6 + drivers/common/dpaax/Makefile | 31 + drivers/common/dpaax/dpaax_iova_table.c | 465 + drivers/common/dpaax/dpaax_iova_table.h | 105 + drivers/common/dpaax/dpaax_logs.h | 39 + drivers/common/dpaax/meson.build | 12 + drivers/common/dpaax/rte_common_dpaax_version.map | 11 + drivers/common/meson.build | 2 +- drivers/common/mvep/Makefile | 38 + drivers/common/mvep/meson.build | 19 + drivers/common/mvep/mvep_common.c | 45 + drivers/common/mvep/rte_common_mvep_version.map | 6 + drivers/common/mvep/rte_mvep_common.h | 21 + drivers/common/qat/qat_common.h | 3 +- drivers/common/qat/qat_device.c | 60 +- drivers/common/qat/qat_device.h | 12 + drivers/common/qat/qat_qp.c | 48 +- drivers/common/qat/qat_qp.h | 1 + drivers/compress/octeontx/include/zip_regs.h | 4 +- drivers/compress/octeontx/otx_zip.h | 6 +- drivers/compress/octeontx/otx_zip_pmd.c | 2 +- drivers/compress/qat/qat_comp.c | 45 +- drivers/compress/qat/qat_comp.h | 13 + drivers/compress/qat/qat_comp_pmd.c | 144 +- drivers/compress/qat/qat_comp_pmd.h | 3 + drivers/crypto/Makefile | 8 +- drivers/crypto/aesni_gcm/aesni_gcm_pmd.c | 88 +- drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c | 8 +- drivers/crypto/aesni_gcm/aesni_gcm_pmd_private.h | 6 +- drivers/crypto/aesni_gcm/meson.build | 12 + drivers/crypto/aesni_mb/aesni_mb_ops.h | 89 +- drivers/crypto/aesni_mb/meson.build | 12 + drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c | 346 +- drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c | 90 +- drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h | 27 +- drivers/crypto/caam_jr/Makefile | 44 + drivers/crypto/caam_jr/caam_jr.c | 2508 ++ drivers/crypto/caam_jr/caam_jr_capabilities.c | 266 + drivers/crypto/caam_jr/caam_jr_capabilities.h | 18 + drivers/crypto/caam_jr/caam_jr_config.h | 207 + drivers/crypto/caam_jr/caam_jr_desc.h | 285 + drivers/crypto/caam_jr/caam_jr_hw.c | 367 + drivers/crypto/caam_jr/caam_jr_hw_specific.h | 503 + drivers/crypto/caam_jr/caam_jr_log.h | 42 + drivers/crypto/caam_jr/caam_jr_pvt.h | 291 + drivers/crypto/caam_jr/caam_jr_uio.c | 501 + drivers/crypto/caam_jr/meson.build | 17 + drivers/crypto/caam_jr/rte_pmd_caam_jr_version.map | 4 + drivers/crypto/dpaa2_sec/Makefile | 11 +- drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c | 788 +- drivers/crypto/dpaa2_sec/dpaa2_sec_event.h | 18 + drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h | 210 +- drivers/crypto/dpaa2_sec/hw/desc.h | 816 +- drivers/crypto/dpaa2_sec/hw/desc/algo.h | 58 +- drivers/crypto/dpaa2_sec/hw/desc/ipsec.h | 195 +- drivers/crypto/dpaa2_sec/hw/desc/pdcp.h | 2796 +++ drivers/crypto/dpaa2_sec/hw/rta/protocol_cmd.h | 346 +- drivers/crypto/dpaa2_sec/hw/rta/sec_run_time_asm.h | 22 + drivers/crypto/dpaa2_sec/mc/dpseci.c | 128 +- drivers/crypto/dpaa2_sec/mc/fsl_dpseci.h | 25 +- drivers/crypto/dpaa2_sec/mc/fsl_dpseci_cmd.h | 73 +- drivers/crypto/dpaa2_sec/meson.build | 2 + .../crypto/dpaa2_sec/rte_pmd_dpaa2_sec_version.map | 8 + drivers/crypto/dpaa_sec/Makefile | 2 +- drivers/crypto/dpaa_sec/dpaa_sec.c | 314 +- drivers/crypto/dpaa_sec/dpaa_sec.h | 3 +- drivers/crypto/kasumi/meson.build | 12 + drivers/crypto/meson.build | 5 +- drivers/crypto/mvsam/Makefile | 5 +- drivers/crypto/mvsam/meson.build | 2 +- drivers/crypto/mvsam/rte_mrvl_pmd.c | 213 +- drivers/crypto/mvsam/rte_mrvl_pmd_ops.c | 160 +- drivers/crypto/mvsam/rte_mrvl_pmd_private.h | 34 +- drivers/crypto/null/null_crypto_pmd_ops.c | 2 +- drivers/crypto/octeontx/Makefile | 46 + drivers/crypto/octeontx/meson.build | 18 + drivers/crypto/octeontx/otx_cryptodev.c | 133 + drivers/crypto/octeontx/otx_cryptodev.h | 20 + .../crypto/octeontx/otx_cryptodev_capabilities.c | 604 + .../crypto/octeontx/otx_cryptodev_capabilities.h | 17 + drivers/crypto/octeontx/otx_cryptodev_hw_access.c | 598 + drivers/crypto/octeontx/otx_cryptodev_hw_access.h | 320 + drivers/crypto/octeontx/otx_cryptodev_mbox.c | 178 + drivers/crypto/octeontx/otx_cryptodev_mbox.h | 92 + drivers/crypto/octeontx/otx_cryptodev_ops.c | 531 + drivers/crypto/octeontx/otx_cryptodev_ops.h | 18 + .../octeontx/rte_pmd_octeontx_crypto_version.map | 4 + drivers/crypto/openssl/compat.h | 265 +- drivers/crypto/openssl/rte_openssl_pmd.c | 29 +- drivers/crypto/openssl/rte_openssl_pmd_ops.c | 40 +- drivers/crypto/qat/qat_sym_capabilities.h | 20 + drivers/crypto/qat/qat_sym_pmd.c | 1 + drivers/crypto/qat/qat_sym_pmd.h | 2 +- drivers/crypto/qat/qat_sym_session.c | 190 +- drivers/crypto/qat/qat_sym_session.h | 3 + drivers/crypto/scheduler/meson.build | 19 + drivers/crypto/scheduler/rte_cryptodev_scheduler.c | 26 +- drivers/crypto/scheduler/rte_cryptodev_scheduler.h | 8 +- drivers/crypto/scheduler/scheduler_failover.c | 6 +- drivers/crypto/scheduler/scheduler_multicore.c | 6 +- .../crypto/scheduler/scheduler_pkt_size_distr.c | 6 +- drivers/crypto/scheduler/scheduler_pmd.c | 8 +- drivers/crypto/scheduler/scheduler_pmd_ops.c | 2 +- drivers/crypto/scheduler/scheduler_pmd_private.h | 2 +- drivers/crypto/scheduler/scheduler_roundrobin.c | 6 +- drivers/crypto/zuc/meson.build | 12 + drivers/event/Makefile | 1 + drivers/event/dpaa/Makefile | 1 + drivers/event/dpaa/dpaa_eventdev.c | 341 +- drivers/event/dpaa/dpaa_eventdev.h | 17 +- drivers/event/dpaa2/Makefile | 8 +- drivers/event/dpaa2/dpaa2_eventdev.c | 462 +- drivers/event/dpaa2/dpaa2_eventdev.h | 24 +- drivers/event/dpaa2/meson.build | 5 +- drivers/event/dsw/Makefile | 29 + drivers/event/dsw/dsw_evdev.c | 435 + drivers/event/dsw/dsw_evdev.h | 279 + drivers/event/dsw/dsw_event.c | 1253 + drivers/event/dsw/dsw_sort.h | 48 + drivers/event/dsw/dsw_xstats.c | 288 + drivers/event/dsw/meson.build | 6 + drivers/event/dsw/rte_pmd_dsw_event_version.map | 3 + drivers/event/meson.build | 2 +- drivers/event/octeontx/Makefile | 2 +- drivers/event/octeontx/ssovf_evdev.c | 83 +- drivers/event/octeontx/ssovf_evdev.h | 5 +- drivers/event/octeontx/ssovf_worker.c | 44 + drivers/event/octeontx/ssovf_worker.h | 1 + drivers/event/opdl/Makefile | 2 +- drivers/event/opdl/meson.build | 11 + ..._version.map => rte_pmd_opdl_event_version.map} | 0 drivers/event/sw/sw_evdev.c | 13 + drivers/event/sw/sw_evdev.h | 8 + drivers/event/sw/sw_evdev_scheduler.c | 20 +- drivers/event/sw/sw_evdev_selftest.c | 77 + drivers/mempool/dpaa/Makefile | 2 +- drivers/mempool/dpaa/dpaa_mempool.c | 8 +- drivers/mempool/dpaa/dpaa_mempool.h | 4 +- drivers/mempool/dpaa2/Makefile | 3 +- drivers/mempool/dpaa2/dpaa2_hw_mempool.c | 38 +- drivers/mempool/dpaa2/meson.build | 2 + .../mempool/dpaa2/rte_mempool_dpaa2_version.map | 1 - drivers/mempool/octeontx/octeontx_fpavf.h | 2 +- drivers/meson.build | 19 +- drivers/net/Makefile | 3 + drivers/net/af_packet/rte_eth_af_packet.c | 12 +- drivers/net/ark/ark_ddm.c | 12 +- drivers/net/ark/ark_ddm.h | 7 +- drivers/net/ark/ark_ethdev.c | 4 +- drivers/net/ark/ark_ethdev_rx.c | 55 +- drivers/net/ark/ark_ethdev_tx.c | 2 +- drivers/net/atlantic/Makefile | 35 + drivers/net/atlantic/atl_common.h | 96 + drivers/net/atlantic/atl_ethdev.c | 1539 ++ drivers/net/atlantic/atl_ethdev.h | 107 + drivers/net/atlantic/atl_hw_regs.c | 52 + drivers/net/atlantic/atl_hw_regs.h | 53 + drivers/net/atlantic/atl_logs.h | 31 + drivers/net/atlantic/atl_rxtx.c | 1357 ++ drivers/net/atlantic/atl_types.h | 186 + drivers/net/atlantic/hw_atl/hw_atl_b0.c | 510 + drivers/net/atlantic/hw_atl/hw_atl_b0.h | 40 + drivers/net/atlantic/hw_atl/hw_atl_b0_internal.h | 145 + drivers/net/atlantic/hw_atl/hw_atl_llh.c | 1490 ++ drivers/net/atlantic/hw_atl/hw_atl_llh.h | 714 + drivers/net/atlantic/hw_atl/hw_atl_llh_internal.h | 2407 ++ drivers/net/atlantic/hw_atl/hw_atl_utils.c | 942 + drivers/net/atlantic/hw_atl/hw_atl_utils.h | 510 + drivers/net/atlantic/hw_atl/hw_atl_utils_fw2x.c | 618 + drivers/net/atlantic/meson.build | 12 + drivers/net/atlantic/rte_pmd_atlantic_version.map | 4 + drivers/net/avf/Makefile | 2 +- drivers/net/avf/avf_ethdev.c | 22 +- drivers/net/avf/avf_rxtx.c | 29 +- drivers/net/avf/avf_rxtx.h | 12 +- drivers/net/avf/avf_rxtx_vec_sse.c | 2 +- drivers/net/avf/avf_vchnl.c | 2 - drivers/net/avf/base/avf_osdep.h | 4 +- drivers/net/avf/base/meson.build | 23 + drivers/net/avf/meson.build | 20 + drivers/net/avp/avp_ethdev.c | 6 - drivers/net/avp/meson.build | 3 + drivers/net/axgbe/axgbe_ethdev.c | 4 - drivers/net/axgbe/axgbe_rxtx.c | 2 +- drivers/net/bnx2x/bnx2x.c | 813 +- drivers/net/bnx2x/bnx2x.h | 42 +- drivers/net/bnx2x/bnx2x_ethdev.c | 92 +- drivers/net/bnx2x/bnx2x_logs.h | 25 +- drivers/net/bnx2x/bnx2x_rxtx.c | 30 +- drivers/net/bnx2x/bnx2x_stats.c | 35 +- drivers/net/bnx2x/bnx2x_vfpf.c | 50 +- drivers/net/bnx2x/ecore_hsi.h | 20 +- drivers/net/bnx2x/ecore_init.h | 6 +- drivers/net/bnx2x/ecore_init_ops.h | 8 +- drivers/net/bnx2x/ecore_reg.h | 1 + drivers/net/bnx2x/ecore_sp.c | 258 +- drivers/net/bnx2x/ecore_sp.h | 4 +- drivers/net/bnx2x/elink.c | 7908 +++--- drivers/net/bnx2x/elink.h | 236 +- drivers/net/bnxt/bnxt.h | 28 +- drivers/net/bnxt/bnxt_cpr.c | 1 + drivers/net/bnxt/bnxt_ethdev.c | 343 +- drivers/net/bnxt/bnxt_filter.c | 28 +- drivers/net/bnxt/bnxt_flow.c | 12 +- drivers/net/bnxt/bnxt_hwrm.c | 418 +- drivers/net/bnxt/bnxt_hwrm.h | 5 + drivers/net/bnxt/bnxt_rxq.c | 25 +- drivers/net/bnxt/bnxt_stats.c | 221 +- drivers/net/bnxt/bnxt_txr.c | 5 +- drivers/net/bnxt/bnxt_vnic.c | 43 +- drivers/net/bnxt/hsi_struct_def_dpdk.h | 24017 +++++++++---------- drivers/net/bonding/Makefile | 1 + drivers/net/bonding/meson.build | 1 + drivers/net/bonding/rte_eth_bond_8023ad.c | 48 +- drivers/net/bonding/rte_eth_bond_8023ad_private.h | 2 +- drivers/net/bonding/rte_eth_bond_api.c | 215 +- drivers/net/bonding/rte_eth_bond_flow.c | 31 +- drivers/net/bonding/rte_eth_bond_pmd.c | 275 +- drivers/net/bonding/rte_eth_bond_private.h | 10 +- drivers/net/cxgbe/Makefile | 2 + drivers/net/cxgbe/base/adapter.h | 4 + drivers/net/cxgbe/base/common.h | 8 + drivers/net/cxgbe/base/t4_hw.c | 108 + drivers/net/cxgbe/base/t4_msg.h | 44 + drivers/net/cxgbe/base/t4_regs.h | 8 + drivers/net/cxgbe/base/t4_tcb.h | 5 + drivers/net/cxgbe/base/t4fw_interface.h | 52 +- drivers/net/cxgbe/cxgbe.h | 15 + drivers/net/cxgbe/cxgbe_ethdev.c | 35 +- drivers/net/cxgbe/cxgbe_filter.c | 117 +- drivers/net/cxgbe/cxgbe_filter.h | 35 + drivers/net/cxgbe/cxgbe_flow.c | 287 +- drivers/net/cxgbe/cxgbe_flow.h | 1 + drivers/net/cxgbe/cxgbe_main.c | 69 +- drivers/net/cxgbe/cxgbevf_main.c | 9 +- drivers/net/cxgbe/l2t.c | 227 + drivers/net/cxgbe/l2t.h | 57 + drivers/net/cxgbe/meson.build | 2 + drivers/net/cxgbe/mps_tcam.c | 243 + drivers/net/cxgbe/mps_tcam.h | 52 + drivers/net/cxgbe/sge.c | 24 +- drivers/net/dpaa/Makefile | 1 + drivers/net/dpaa/dpaa_ethdev.c | 152 +- drivers/net/dpaa/dpaa_ethdev.h | 5 +- drivers/net/dpaa/dpaa_rxtx.c | 100 +- drivers/net/dpaa/dpaa_rxtx.h | 5 +- drivers/net/dpaa2/Makefile | 3 +- drivers/net/dpaa2/base/dpaa2_hw_dpni_annot.h | 40 + drivers/net/dpaa2/dpaa2_ethdev.c | 192 +- drivers/net/dpaa2/dpaa2_rxtx.c | 95 +- drivers/net/dpaa2/mc/dpni.c | 134 +- drivers/net/dpaa2/mc/fsl_dpkg.h | 71 +- drivers/net/dpaa2/mc/fsl_dpni.h | 378 +- drivers/net/dpaa2/mc/fsl_dpni_cmd.h | 87 +- drivers/net/dpaa2/mc/fsl_net.h | 2 +- drivers/net/dpaa2/meson.build | 2 + drivers/net/e1000/Makefile | 1 + drivers/net/e1000/base/e1000_82571.c | 5 + drivers/net/e1000/base/e1000_osdep.h | 2 +- drivers/net/e1000/base/meson.build | 3 + drivers/net/e1000/e1000_ethdev.h | 6 +- drivers/net/e1000/em_ethdev.c | 8 +- drivers/net/e1000/em_rxtx.c | 15 +- drivers/net/e1000/igb_ethdev.c | 43 +- drivers/net/e1000/igb_flow.c | 10 +- drivers/net/e1000/igb_rxtx.c | 31 +- drivers/net/e1000/meson.build | 2 + drivers/net/ena/ena_ethdev.c | 202 +- drivers/net/enetc/Makefile | 23 + drivers/net/enetc/base/enetc_hw.h | 226 + drivers/net/enetc/enetc.h | 113 + drivers/net/enetc/enetc_ethdev.c | 629 + drivers/net/enetc/enetc_logs.h | 40 + drivers/net/enetc/enetc_rxtx.c | 239 + drivers/net/enetc/meson.build | 11 + drivers/net/enetc/rte_pmd_enetc_version.map | 4 + drivers/net/enic/Makefile | 28 + drivers/net/enic/base/vnic_dev.c | 105 + drivers/net/enic/base/vnic_dev.h | 8 + drivers/net/enic/base/vnic_devcmd.h | 72 +- drivers/net/enic/enic.h | 12 + drivers/net/enic/enic_ethdev.c | 61 +- drivers/net/enic/enic_flow.c | 180 +- drivers/net/enic/enic_main.c | 81 +- drivers/net/enic/enic_res.c | 11 +- drivers/net/enic/enic_rxtx.c | 286 +- drivers/net/enic/enic_rxtx_common.h | 271 + drivers/net/enic/enic_rxtx_vec_avx2.c | 831 + drivers/net/enic/meson.build | 16 + drivers/net/failsafe/failsafe.c | 11 +- drivers/net/failsafe/failsafe_args.c | 10 +- drivers/net/failsafe/failsafe_eal.c | 3 +- drivers/net/failsafe/failsafe_ether.c | 116 +- drivers/net/failsafe/failsafe_flow.c | 31 +- drivers/net/failsafe/failsafe_intr.c | 2 +- drivers/net/failsafe/failsafe_ops.c | 252 +- drivers/net/failsafe/failsafe_private.h | 15 +- drivers/net/failsafe/failsafe_rxtx.c | 2 +- drivers/net/fm10k/base/meson.build | 3 + drivers/net/fm10k/fm10k_ethdev.c | 45 +- drivers/net/i40e/base/README | 2 +- drivers/net/i40e/base/i40e_adminq.c | 50 +- drivers/net/i40e/base/i40e_adminq.h | 35 +- drivers/net/i40e/base/i40e_adminq_cmd.h | 234 +- drivers/net/i40e/base/i40e_alloc.h | 35 +- drivers/net/i40e/base/i40e_common.c | 250 +- drivers/net/i40e/base/i40e_dcb.c | 46 +- drivers/net/i40e/base/i40e_dcb.h | 35 +- drivers/net/i40e/base/i40e_devids.h | 38 +- drivers/net/i40e/base/i40e_diag.c | 37 +- drivers/net/i40e/base/i40e_diag.h | 37 +- drivers/net/i40e/base/i40e_hmc.c | 35 +- drivers/net/i40e/base/i40e_hmc.h | 35 +- drivers/net/i40e/base/i40e_lan_hmc.c | 50 +- drivers/net/i40e/base/i40e_lan_hmc.h | 35 +- drivers/net/i40e/base/i40e_nvm.c | 35 +- drivers/net/i40e/base/i40e_osdep.h | 40 +- drivers/net/i40e/base/i40e_prototype.h | 76 +- drivers/net/i40e/base/i40e_register.h | 35 +- drivers/net/i40e/base/i40e_status.h | 35 +- drivers/net/i40e/base/i40e_type.h | 58 +- drivers/net/i40e/base/meson.build | 7 +- drivers/net/i40e/base/virtchnl.h | 35 +- drivers/net/i40e/i40e_ethdev.c | 241 +- drivers/net/i40e/i40e_ethdev.h | 7 +- drivers/net/i40e/i40e_ethdev_vf.c | 26 +- drivers/net/i40e/i40e_flow.c | 49 +- drivers/net/i40e/i40e_rxtx.c | 172 +- drivers/net/i40e/i40e_rxtx_vec_common.h | 4 +- drivers/net/i40e/i40e_vf_representor.c | 12 +- drivers/net/i40e/rte_pmd_i40e.c | 2 +- drivers/net/ifc/base/ifcvf.c | 33 +- drivers/net/ifc/base/ifcvf.h | 7 + drivers/net/ifc/base/ifcvf_osdep.h | 2 +- drivers/net/ifc/ifcvf_vdpa.c | 102 +- drivers/net/ixgbe/base/README | 4 +- drivers/net/ixgbe/base/ixgbe_82598.c | 35 +- drivers/net/ixgbe/base/ixgbe_82598.h | 35 +- drivers/net/ixgbe/base/ixgbe_82599.c | 45 +- drivers/net/ixgbe/base/ixgbe_82599.h | 35 +- drivers/net/ixgbe/base/ixgbe_api.c | 50 +- drivers/net/ixgbe/base/ixgbe_api.h | 36 +- drivers/net/ixgbe/base/ixgbe_common.c | 43 +- drivers/net/ixgbe/base/ixgbe_common.h | 35 +- drivers/net/ixgbe/base/ixgbe_dcb.c | 35 +- drivers/net/ixgbe/base/ixgbe_dcb.h | 35 +- drivers/net/ixgbe/base/ixgbe_dcb_82598.c | 35 +- drivers/net/ixgbe/base/ixgbe_dcb_82598.h | 35 +- drivers/net/ixgbe/base/ixgbe_dcb_82599.c | 35 +- drivers/net/ixgbe/base/ixgbe_dcb_82599.h | 35 +- drivers/net/ixgbe/base/ixgbe_hv_vf.c | 35 +- drivers/net/ixgbe/base/ixgbe_hv_vf.h | 35 +- drivers/net/ixgbe/base/ixgbe_mbx.c | 35 +- drivers/net/ixgbe/base/ixgbe_mbx.h | 35 +- drivers/net/ixgbe/base/ixgbe_osdep.h | 38 +- drivers/net/ixgbe/base/ixgbe_phy.c | 36 +- drivers/net/ixgbe/base/ixgbe_phy.h | 35 +- drivers/net/ixgbe/base/ixgbe_type.h | 53 +- drivers/net/ixgbe/base/ixgbe_vf.c | 35 +- drivers/net/ixgbe/base/ixgbe_vf.h | 35 +- drivers/net/ixgbe/base/ixgbe_x540.c | 35 +- drivers/net/ixgbe/base/ixgbe_x540.h | 35 +- drivers/net/ixgbe/base/ixgbe_x550.c | 59 +- drivers/net/ixgbe/base/ixgbe_x550.h | 36 +- drivers/net/ixgbe/base/meson.build | 3 + drivers/net/ixgbe/ixgbe_ethdev.c | 42 +- drivers/net/ixgbe/ixgbe_flow.c | 29 + drivers/net/ixgbe/ixgbe_ipsec.c | 6 +- drivers/net/ixgbe/ixgbe_rxtx.c | 32 +- drivers/net/ixgbe/ixgbe_vf_representor.c | 10 +- drivers/net/ixgbe/meson.build | 3 +- drivers/net/kni/rte_eth_kni.c | 12 +- drivers/net/liquidio/lio_ethdev.c | 5 +- drivers/net/meson.build | 14 +- drivers/net/mlx4/meson.build | 102 + drivers/net/mlx4/mlx4.c | 8 +- drivers/net/mlx4/mlx4_mr.c | 149 + drivers/net/mlx4/mlx4_rxq.c | 3 +- drivers/net/mlx4/mlx4_rxtx.h | 35 +- drivers/net/mlx5/Makefile | 43 +- drivers/net/mlx5/meson.build | 244 + drivers/net/mlx5/mlx5.c | 154 +- drivers/net/mlx5/mlx5.h | 51 +- drivers/net/mlx5/mlx5_ethdev.c | 72 +- drivers/net/mlx5/mlx5_flow.c | 3691 ++- drivers/net/mlx5/mlx5_flow.h | 375 + drivers/net/mlx5/mlx5_flow_dv.c | 1492 ++ drivers/net/mlx5/mlx5_flow_tcf.c | 2913 +++ drivers/net/mlx5/mlx5_flow_verbs.c | 1825 ++ drivers/net/mlx5/mlx5_glue.c | 113 +- drivers/net/mlx5/mlx5_glue.h | 34 +- drivers/net/mlx5/mlx5_mac.c | 2 +- drivers/net/mlx5/mlx5_mr.c | 155 + drivers/net/mlx5/mlx5_nl_flow.c | 1248 - drivers/net/mlx5/mlx5_prm.h | 222 +- drivers/net/mlx5/mlx5_rxq.c | 3 +- drivers/net/mlx5/mlx5_rxtx.c | 48 +- drivers/net/mlx5/mlx5_rxtx.h | 41 +- drivers/net/mlx5/mlx5_rxtx_vec.c | 46 +- drivers/net/mlx5/mlx5_rxtx_vec.h | 1 + drivers/net/mlx5/mlx5_rxtx_vec_neon.h | 11 +- drivers/net/mlx5/mlx5_rxtx_vec_sse.h | 10 +- drivers/net/mlx5/mlx5_socket.c | 2 - drivers/net/mlx5/mlx5_stats.c | 87 +- drivers/net/mlx5/mlx5_txq.c | 5 +- drivers/net/mvneta/Makefile | 42 + drivers/net/mvneta/meson.build | 28 + drivers/net/mvneta/mvneta_ethdev.c | 987 + drivers/net/mvneta/mvneta_ethdev.h | 79 + drivers/net/mvneta/mvneta_rxtx.c | 1030 + drivers/net/mvneta/mvneta_rxtx.h | 38 + drivers/net/mvneta/rte_pmd_mvneta_version.map | 3 + drivers/net/mvpp2/Makefile | 5 +- drivers/net/mvpp2/meson.build | 6 +- drivers/net/mvpp2/mrvl_ethdev.c | 510 +- drivers/net/mvpp2/mrvl_ethdev.h | 123 +- drivers/net/mvpp2/mrvl_flow.c | 161 +- drivers/net/mvpp2/mrvl_flow.h | 15 + drivers/net/mvpp2/mrvl_mtr.c | 512 + drivers/net/mvpp2/mrvl_mtr.h | 15 + drivers/net/mvpp2/mrvl_qos.c | 246 +- drivers/net/mvpp2/mrvl_qos.h | 2 +- drivers/net/mvpp2/mrvl_tm.c | 1009 + drivers/net/mvpp2/mrvl_tm.h | 15 + drivers/net/netvsc/Makefile | 1 + drivers/net/netvsc/hn_ethdev.c | 221 +- drivers/net/netvsc/hn_nvs.c | 24 +- drivers/net/netvsc/hn_nvs.h | 9 + drivers/net/netvsc/hn_rndis.c | 47 +- drivers/net/netvsc/hn_rndis.h | 3 +- drivers/net/netvsc/hn_rxtx.c | 212 +- drivers/net/netvsc/hn_var.h | 66 +- drivers/net/netvsc/hn_vf.c | 549 + drivers/net/netvsc/meson.build | 2 +- drivers/net/nfp/Makefile | 1 + drivers/net/nfp/meson.build | 5 + drivers/net/nfp/nfp_net.c | 77 +- drivers/net/nfp/nfp_net_pmd.h | 2 + drivers/net/null/rte_eth_null.c | 8 +- drivers/net/octeontx/base/meson.build | 6 +- drivers/net/octeontx/base/octeontx_io.h | 2 +- drivers/net/octeontx/octeontx_ethdev.c | 45 +- drivers/net/octeontx/octeontx_ethdev.h | 3 +- drivers/net/octeontx/octeontx_rxtx.c | 35 +- drivers/net/octeontx/octeontx_rxtx.h | 33 + drivers/net/pcap/rte_eth_pcap.c | 186 +- drivers/net/qede/Makefile | 2 +- drivers/net/qede/base/bcm_osal.c | 2 +- drivers/net/qede/base/bcm_osal.h | 3 + drivers/net/qede/base/common_hsi.h | 15 +- drivers/net/qede/base/ecore.h | 62 +- drivers/net/qede/base/ecore_cxt.c | 15 +- drivers/net/qede/base/ecore_dcbx.c | 99 +- drivers/net/qede/base/ecore_dcbx_api.h | 10 + drivers/net/qede/base/ecore_dev.c | 1896 +- drivers/net/qede/base/ecore_dev_api.h | 173 +- drivers/net/qede/base/ecore_hsi_common.h | 57 +- drivers/net/qede/base/ecore_hsi_debug_tools.h | 15 + drivers/net/qede/base/ecore_hsi_eth.h | 57 +- drivers/net/qede/base/ecore_hw.c | 127 +- drivers/net/qede/base/ecore_hw.h | 40 +- drivers/net/qede/base/ecore_init_fw_funcs.c | 93 +- drivers/net/qede/base/ecore_init_fw_funcs.h | 42 +- drivers/net/qede/base/ecore_init_ops.c | 26 +- drivers/net/qede/base/ecore_int.c | 99 +- drivers/net/qede/base/ecore_int.h | 1 + drivers/net/qede/base/ecore_int_api.h | 14 +- drivers/net/qede/base/ecore_iov_api.h | 17 + drivers/net/qede/base/ecore_iro.h | 164 +- drivers/net/qede/base/ecore_iro_values.h | 42 +- drivers/net/qede/base/ecore_l2.c | 108 +- drivers/net/qede/base/ecore_l2_api.h | 41 +- drivers/net/qede/base/ecore_mcp.c | 280 +- drivers/net/qede/base/ecore_mcp.h | 21 +- drivers/net/qede/base/ecore_mcp_api.h | 40 +- drivers/net/qede/base/ecore_rt_defs.h | 265 +- drivers/net/qede/base/ecore_sp_commands.c | 8 +- drivers/net/qede/base/ecore_sp_commands.h | 3 + drivers/net/qede/base/ecore_spq.c | 56 +- drivers/net/qede/base/ecore_sriov.c | 94 +- drivers/net/qede/base/ecore_vf.c | 25 +- drivers/net/qede/base/ecore_vfpf_if.h | 2 +- drivers/net/qede/base/eth_common.h | 5 + drivers/net/qede/base/mcp_public.h | 44 + drivers/net/qede/base/meson.build | 60 + drivers/net/qede/base/reg_addr.h | 76 +- drivers/net/qede/meson.build | 12 + drivers/net/qede/qede_ethdev.c | 743 +- drivers/net/qede/qede_ethdev.h | 69 +- drivers/net/qede/qede_fdir.c | 470 - drivers/net/qede/qede_filter.c | 1546 ++ drivers/net/qede/qede_main.c | 15 +- drivers/net/qede/qede_rxtx.c | 140 +- drivers/net/qede/qede_rxtx.h | 17 +- drivers/net/ring/rte_eth_ring.c | 7 +- drivers/net/sfc/base/ef10_ev.c | 38 +- drivers/net/sfc/base/ef10_filter.c | 64 +- drivers/net/sfc/base/ef10_image.c | 5 +- drivers/net/sfc/base/ef10_impl.h | 68 +- drivers/net/sfc/base/ef10_intr.c | 5 +- drivers/net/sfc/base/ef10_mac.c | 30 +- drivers/net/sfc/base/ef10_nic.c | 277 +- drivers/net/sfc/base/ef10_nvram.c | 161 +- drivers/net/sfc/base/ef10_phy.c | 105 +- drivers/net/sfc/base/ef10_rx.c | 89 +- drivers/net/sfc/base/ef10_signed_image_layout.h | 8 + drivers/net/sfc/base/ef10_tx.c | 38 +- drivers/net/sfc/base/efx.h | 353 +- drivers/net/sfc/base/efx_annote.h | 103 + drivers/net/sfc/base/efx_bootcfg.c | 641 +- drivers/net/sfc/base/efx_filter.c | 90 +- drivers/net/sfc/base/efx_impl.h | 10 +- drivers/net/sfc/base/efx_lic.c | 91 +- drivers/net/sfc/base/efx_mcdi.c | 168 +- drivers/net/sfc/base/efx_mcdi.h | 19 +- drivers/net/sfc/base/efx_mon.c | 745 +- drivers/net/sfc/base/efx_nic.c | 54 +- drivers/net/sfc/base/efx_nvram.c | 71 +- drivers/net/sfc/base/efx_phy.c | 60 +- drivers/net/sfc/base/efx_port.c | 2 +- drivers/net/sfc/base/efx_rx.c | 225 +- drivers/net/sfc/base/efx_tunnel.c | 6 +- drivers/net/sfc/base/efx_tx.c | 19 +- drivers/net/sfc/base/hunt_nic.c | 19 +- drivers/net/sfc/base/mc_driver_pcol_strs.h | 102 + drivers/net/sfc/base/mcdi_mon.c | 381 +- drivers/net/sfc/base/mcdi_mon.h | 5 + drivers/net/sfc/base/medford2_nic.c | 15 +- drivers/net/sfc/base/medford_nic.c | 15 +- drivers/net/sfc/base/meson.build | 3 + drivers/net/sfc/base/siena_mac.c | 9 +- drivers/net/sfc/base/siena_nic.c | 7 +- drivers/net/sfc/base/siena_nvram.c | 5 +- drivers/net/sfc/base/siena_phy.c | 28 +- drivers/net/sfc/efsys.h | 38 - drivers/net/sfc/sfc_dp_tx.h | 5 + drivers/net/sfc/sfc_ef10_essb_rx.c | 16 +- drivers/net/sfc/sfc_ef10_rx.c | 227 +- drivers/net/sfc/sfc_ef10_tx.c | 364 +- drivers/net/sfc/sfc_ethdev.c | 25 +- drivers/net/sfc/sfc_rx.c | 33 +- drivers/net/sfc/sfc_tso.c | 25 +- drivers/net/sfc/sfc_tso.h | 23 + drivers/net/sfc/sfc_tx.c | 12 +- drivers/net/softnic/Makefile | 4 + drivers/net/softnic/conn.c | 1 - drivers/net/softnic/hash_func.h | 359 - drivers/net/softnic/hash_func_arm64.h | 261 - drivers/net/softnic/meson.build | 8 +- drivers/net/softnic/rte_eth_softnic.c | 42 +- drivers/net/softnic/rte_eth_softnic_action.c | 67 +- drivers/net/softnic/rte_eth_softnic_cli.c | 883 +- drivers/net/softnic/rte_eth_softnic_cryptodev.c | 125 + drivers/net/softnic/rte_eth_softnic_flow.c | 2287 ++ drivers/net/softnic/rte_eth_softnic_internals.h | 219 +- drivers/net/softnic/rte_eth_softnic_meter.c | 728 + drivers/net/softnic/rte_eth_softnic_pipeline.c | 161 +- drivers/net/softnic/rte_eth_softnic_thread.c | 118 +- drivers/net/szedata2/rte_eth_szedata2.c | 10 +- drivers/net/tap/Makefile | 1 + drivers/net/tap/meson.build | 45 + drivers/net/tap/rte_eth_tap.c | 269 +- drivers/net/tap/rte_eth_tap.h | 10 +- drivers/net/tap/tap_bpf_insns.h | 4 +- drivers/net/tap/tap_flow.c | 5 +- drivers/net/tap/tap_intr.c | 5 +- drivers/net/thunderx/base/meson.build | 6 +- drivers/net/thunderx/nicvf_ethdev.c | 9 - drivers/net/thunderx/nicvf_ethdev.h | 1 - drivers/net/vdev_netvsc/meson.build | 22 + drivers/net/vdev_netvsc/vdev_netvsc.c | 2 +- drivers/net/vhost/rte_eth_vhost.c | 31 +- drivers/net/virtio/virtio_ethdev.c | 8 +- drivers/net/virtio/virtio_ethdev.h | 5 +- drivers/net/virtio/virtio_pci.c | 65 +- drivers/net/virtio/virtio_rxtx_simple.c | 2 +- drivers/net/virtio/virtio_user/vhost.h | 4 +- drivers/net/virtio/virtio_user/vhost_kernel.c | 65 +- drivers/net/virtio/virtio_user/vhost_kernel_tap.c | 56 +- drivers/net/virtio/virtio_user/vhost_kernel_tap.h | 2 +- drivers/net/virtio/virtio_user/vhost_user.c | 179 +- drivers/net/virtio/virtio_user/virtio_user_dev.c | 35 +- drivers/net/virtio/virtio_user_ethdev.c | 4 - drivers/net/vmxnet3/meson.build | 18 + drivers/net/vmxnet3/vmxnet3_ethdev.c | 88 +- drivers/net/vmxnet3/vmxnet3_ethdev.h | 3 + drivers/raw/dpaa2_cmdif/Makefile | 3 +- drivers/raw/dpaa2_cmdif/meson.build | 2 + drivers/raw/dpaa2_qdma/Makefile | 3 +- drivers/raw/dpaa2_qdma/dpaa2_qdma.c | 18 +- drivers/raw/dpaa2_qdma/dpaa2_qdma.h | 6 +- drivers/raw/dpaa2_qdma/meson.build | 2 + drivers/raw/ifpga_rawdev/base/ifpga_enumerate.c | 8 +- drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.h | 8 +- drivers/raw/ifpga_rawdev/base/ifpga_port.c | 6 +- drivers/raw/ifpga_rawdev/base/ifpga_port_error.c | 2 +- drivers/raw/ifpga_rawdev/base/meson.build | 4 +- drivers/raw/ifpga_rawdev/ifpga_rawdev.c | 5 +- drivers/raw/skeleton_rawdev/skeleton_rawdev.c | 2 +- drivers/raw/skeleton_rawdev/skeleton_rawdev_test.c | 3 +- examples/Makefile | 2 +- examples/bbdev_app/main.c | 1 - examples/bond/main.c | 1 - examples/cmdline/Makefile | 1 - examples/ethtool/ethtool-app/Makefile | 2 +- examples/eventdev_pipeline/main.c | 88 +- examples/eventdev_pipeline/pipeline_common.h | 31 +- .../eventdev_pipeline/pipeline_worker_generic.c | 268 +- examples/eventdev_pipeline/pipeline_worker_tx.c | 156 +- examples/exception_path/main.c | 3 - examples/flow_filtering/main.c | 17 - examples/ip_fragmentation/main.c | 3 +- examples/ip_pipeline/Makefile | 1 + examples/ip_pipeline/action.c | 67 +- examples/ip_pipeline/action.h | 1 + examples/ip_pipeline/cli.c | 831 +- examples/ip_pipeline/conn.c | 1 - examples/ip_pipeline/cryptodev.c | 117 + examples/ip_pipeline/cryptodev.h | 43 + examples/ip_pipeline/examples/flow_crypto.cli | 58 + examples/ip_pipeline/hash_func.h | 357 - examples/ip_pipeline/hash_func_arm64.h | 232 - examples/ip_pipeline/link.c | 1 - examples/ip_pipeline/main.c | 9 + examples/ip_pipeline/meson.build | 3 +- examples/ip_pipeline/pipeline.c | 79 +- examples/ip_pipeline/pipeline.h | 16 + examples/ip_pipeline/thread.c | 70 +- examples/ip_reassembly/main.c | 3 +- examples/ipsec-secgw/esp.c | 3 + examples/ipsec-secgw/ipsec-secgw.c | 23 +- examples/ipsec-secgw/sa.c | 11 +- examples/ipv4_multicast/main.c | 5 +- examples/kni/Makefile | 2 + examples/kni/main.c | 95 +- examples/kni/meson.build | 1 + examples/l2fwd-cat/Makefile | 2 - examples/l2fwd-cat/meson.build | 1 - examples/l2fwd-crypto/main.c | 1 - examples/l2fwd-jobstats/main.c | 1 - examples/l2fwd-keepalive/main.c | 1 - examples/l2fwd/main.c | 1 - examples/l3fwd-acl/main.c | 3 +- examples/l3fwd-power/Makefile | 3 + examples/l3fwd-power/main.c | 345 +- examples/l3fwd-power/meson.build | 1 + examples/l3fwd-vf/main.c | 3 +- examples/l3fwd/main.c | 3 +- examples/link_status_interrupt/main.c | 1 - examples/load_balancer/Makefile | 1 - examples/load_balancer/init.c | 3 +- examples/meson.build | 4 + examples/multi_process/Makefile | 1 + examples/multi_process/hotplug_mp/Makefile | 23 + examples/multi_process/hotplug_mp/commands.c | 214 + examples/multi_process/hotplug_mp/commands.h | 10 + examples/multi_process/hotplug_mp/main.c | 41 + examples/multi_process/symmetric_mp/main.c | 3 +- examples/netmap_compat/bridge/bridge.c | 1 - examples/performance-thread/l3fwd-thread/main.c | 5 +- examples/performance-thread/pthread_shim/main.c | 1 - .../performance-thread/pthread_shim/pthread_shim.c | 1 - examples/qos_meter/main.c | 3 +- examples/qos_sched/Makefile | 2 - examples/qos_sched/init.c | 1 - examples/quota_watermark/qw/init.c | 1 - examples/service_cores/main.c | 6 + examples/tep_termination/Makefile | 1 - examples/tep_termination/vxlan_setup.c | 1 - examples/vdpa/Makefile | 32 + examples/vdpa/main.c | 454 + examples/vdpa/meson.build | 16 + examples/vhost/Makefile | 1 - examples/vhost/main.c | 6 +- examples/vhost_crypto/Makefile | 1 - examples/vhost_crypto/main.c | 480 +- examples/vhost_crypto/meson.build | 2 +- examples/vhost_scsi/Makefile | 4 +- examples/vhost_scsi/meson.build | 2 +- examples/vm_power_manager/Makefile | 6 + examples/vm_power_manager/channel_manager.c | 180 +- examples/vm_power_manager/channel_manager.h | 21 +- examples/vm_power_manager/channel_monitor.c | 532 +- examples/vm_power_manager/guest_cli/meson.build | 21 + .../guest_cli/vm_power_cli_guest.c | 1 + examples/vm_power_manager/main.c | 2 + examples/vm_power_manager/meson.build | 37 +- kernel/linux/igb_uio/igb_uio.c | 35 +- kernel/linux/igb_uio/meson.build | 2 + kernel/linux/kni/Kbuild | 8 + kernel/linux/kni/ethtool/igb/igb_ethtool.c | 8 + kernel/linux/kni/ethtool/igb/meson.build | 16 + kernel/linux/kni/ethtool/ixgbe/ixgbe.h | 2 + kernel/linux/kni/ethtool/ixgbe/ixgbe_ethtool.c | 8 + kernel/linux/kni/ethtool/ixgbe/meson.build | 13 + kernel/linux/kni/ethtool/meson.build | 5 + kernel/linux/kni/kni_dev.h | 3 + kernel/linux/kni/kni_ethtool.c | 10 + kernel/linux/kni/kni_fifo.h | 24 +- kernel/linux/kni/kni_misc.c | 60 +- kernel/linux/kni/kni_net.c | 7 +- kernel/linux/kni/meson.build | 34 + kernel/linux/meson.build | 2 +- lib/Makefile | 7 +- lib/librte_acl/rte_acl.c | 8 +- lib/librte_acl/rte_acl.h | 2 +- lib/librte_bpf/bpf_load.c | 2 +- lib/librte_bpf/rte_bpf_ethdev.h | 2 +- lib/librte_cmdline/Makefile | 1 - lib/librte_cmdline/cmdline.c | 24 - lib/librte_cmdline/meson.build | 4 + lib/librte_compressdev/rte_comp.c | 4 +- lib/librte_compressdev/rte_comp.h | 4 +- lib/librte_compressdev/rte_compressdev.c | 32 +- lib/librte_compressdev/rte_compressdev_pmd.c | 23 +- lib/librte_compressdev/rte_compressdev_pmd.h | 5 - lib/librte_cryptodev/Makefile | 2 +- lib/librte_cryptodev/meson.build | 2 +- lib/librte_cryptodev/rte_cryptodev.c | 37 +- lib/librte_cryptodev/rte_cryptodev_pmd.c | 23 +- lib/librte_cryptodev/rte_cryptodev_pmd.h | 3 - lib/librte_eal/bsdapp/eal/Makefile | 9 +- lib/librte_eal/bsdapp/eal/eal.c | 54 +- lib/librte_eal/bsdapp/eal/eal_dev.c | 14 + lib/librte_eal/bsdapp/eal/eal_memalloc.c | 21 + lib/librte_eal/bsdapp/eal/eal_memory.c | 9 +- lib/librte_eal/common/Makefile | 1 + lib/librte_eal/common/arch/arm/meson.build | 2 +- lib/librte_eal/common/arch/ppc_64/meson.build | 5 + lib/librte_eal/common/arch/x86/meson.build | 2 +- lib/librte_eal/common/eal_common_bus.c | 45 +- lib/librte_eal/common/eal_common_class.c | 2 +- lib/librte_eal/common/eal_common_dev.c | 347 +- lib/librte_eal/common/eal_common_devargs.c | 52 +- lib/librte_eal/common/eal_common_fbarray.c | 5 + lib/librte_eal/common/eal_common_memory.c | 210 +- lib/librte_eal/common/eal_common_memzone.c | 8 +- lib/librte_eal/common/eal_common_options.c | 42 +- lib/librte_eal/common/eal_common_proc.c | 10 +- lib/librte_eal/common/eal_common_string_fns.c | 26 + lib/librte_eal/common/eal_common_timer.c | 24 + lib/librte_eal/common/eal_filesystem.h | 15 +- lib/librte_eal/common/eal_internal_cfg.h | 1 + lib/librte_eal/common/eal_memalloc.h | 11 + lib/librte_eal/common/eal_options.h | 2 + lib/librte_eal/common/eal_private.h | 90 +- lib/librte_eal/common/hotplug_mp.c | 426 + lib/librte_eal/common/hotplug_mp.h | 46 + .../common/include/arch/arm/rte_cycles_32.h | 4 +- .../common/include/arch/ppc_64/meson.build | 16 + .../common/include/arch/ppc_64/rte_pause.h | 7 + lib/librte_eal/common/include/generic/rte_cycles.h | 11 + lib/librte_eal/common/include/rte_bitmap.h | 14 +- lib/librte_eal/common/include/rte_bus.h | 34 + lib/librte_eal/common/include/rte_common.h | 11 + lib/librte_eal/common/include/rte_dev.h | 123 +- lib/librte_eal/common/include/rte_devargs.h | 81 +- lib/librte_eal/common/include/rte_eal.h | 20 +- lib/librte_eal/common/include/rte_eal_interrupts.h | 1 + lib/librte_eal/common/include/rte_eal_memconfig.h | 18 +- lib/librte_eal/common/include/rte_malloc.h | 192 + lib/librte_eal/common/include/rte_malloc_heap.h | 3 + lib/librte_eal/common/include/rte_memory.h | 109 + lib/librte_eal/common/include/rte_option.h | 63 + lib/librte_eal/common/include/rte_string_fns.h | 26 +- lib/librte_eal/common/include/rte_version.h | 6 +- lib/librte_eal/common/include/rte_vfio.h | 31 +- lib/librte_eal/common/malloc_elem.c | 10 +- lib/librte_eal/common/malloc_heap.c | 340 +- lib/librte_eal/common/malloc_heap.h | 17 + lib/librte_eal/common/malloc_mp.c | 4 +- lib/librte_eal/common/meson.build | 5 + lib/librte_eal/common/rte_malloc.c | 436 +- lib/librte_eal/common/rte_option.c | 54 + lib/librte_eal/linuxapp/eal/Makefile | 20 +- lib/librte_eal/linuxapp/eal/eal.c | 119 +- lib/librte_eal/linuxapp/eal/eal_dev.c | 172 +- lib/librte_eal/linuxapp/eal/eal_hugepage_info.c | 1 + lib/librte_eal/linuxapp/eal/eal_interrupts.c | 79 + lib/librte_eal/linuxapp/eal/eal_memalloc.c | 466 +- lib/librte_eal/linuxapp/eal/eal_memory.c | 264 +- lib/librte_eal/linuxapp/eal/eal_thread.c | 4 +- lib/librte_eal/linuxapp/eal/eal_timer.c | 5 +- lib/librte_eal/linuxapp/eal/eal_vfio.c | 216 +- lib/librte_eal/linuxapp/eal/eal_vfio.h | 4 + lib/librte_eal/linuxapp/eal/eal_vfio_mp_sync.c | 11 + .../linuxapp/eal/include/exec-env/rte_kni_common.h | 6 + lib/librte_eal/meson.build | 3 +- lib/librte_eal/rte_eal_version.map | 39 +- lib/librte_ethdev/Makefile | 6 +- lib/librte_ethdev/ethdev_private.c | 121 + lib/librte_ethdev/ethdev_private.h | 38 + lib/librte_ethdev/ethdev_profile.c | 103 +- lib/librte_ethdev/ethdev_profile.h | 6 +- lib/librte_ethdev/meson.build | 8 +- lib/librte_ethdev/rte_class_eth.c | 173 + lib/librte_ethdev/rte_ethdev.c | 463 +- lib/librte_ethdev/rte_ethdev.h | 183 +- lib/librte_ethdev/rte_ethdev_core.h | 50 +- lib/librte_ethdev/rte_ethdev_driver.h | 37 +- lib/librte_ethdev/rte_ethdev_pci.h | 11 - lib/librte_ethdev/rte_ethdev_version.map | 17 +- lib/librte_ethdev/rte_flow.c | 686 +- lib/librte_ethdev/rte_flow.h | 484 +- lib/librte_ethdev/rte_tm.h | 4 +- lib/librte_eventdev/Makefile | 4 +- lib/librte_eventdev/meson.build | 8 +- lib/librte_eventdev/rte_event_eth_rx_adapter.c | 6 +- lib/librte_eventdev/rte_event_eth_rx_adapter.h | 4 - lib/librte_eventdev/rte_event_eth_tx_adapter.c | 1138 + lib/librte_eventdev/rte_event_eth_tx_adapter.h | 462 + lib/librte_eventdev/rte_eventdev.c | 70 +- lib/librte_eventdev/rte_eventdev.h | 74 +- lib/librte_eventdev/rte_eventdev_pmd.h | 225 +- lib/librte_eventdev/rte_eventdev_version.map | 13 + lib/librte_flow_classify/rte_flow_classify.c | 3 +- lib/librte_hash/rte_cuckoo_hash.c | 1061 +- lib/librte_hash/rte_cuckoo_hash.h | 34 +- lib/librte_hash/rte_hash.h | 85 +- lib/librte_hash/rte_hash_version.map | 7 + lib/librte_ip_frag/ip_frag_common.h | 23 +- lib/librte_ip_frag/ip_frag_internal.c | 18 - lib/librte_ip_frag/rte_ip_frag.h | 19 +- lib/librte_ip_frag/rte_ip_frag_common.c | 21 + lib/librte_ip_frag/rte_ip_frag_version.map | 6 + lib/librte_kni/rte_kni.c | 534 +- lib/librte_kni/rte_kni.h | 26 +- lib/librte_kni/rte_kni_fifo.h | 43 +- lib/librte_kni/rte_kni_version.map | 6 + lib/librte_kvargs/rte_kvargs.c | 17 + lib/librte_kvargs/rte_kvargs.h | 7 +- lib/librte_latencystats/rte_latencystats.c | 7 +- lib/librte_lpm/Makefile | 2 +- lib/librte_lpm/meson.build | 1 + lib/librte_lpm/rte_lpm6.c | 1050 +- lib/librte_mbuf/meson.build | 2 +- lib/librte_mbuf/rte_mbuf.c | 53 +- lib/librte_mbuf/rte_mbuf.h | 130 +- lib/librte_mbuf/rte_mbuf_ptype.c | 3 + lib/librte_mbuf/rte_mbuf_ptype.h | 22 + lib/librte_mempool/rte_mempool.c | 57 +- lib/librte_net/Makefile | 2 +- lib/librte_net/meson.build | 3 +- lib/librte_net/net_crc_sse.h | 4 +- lib/librte_net/rte_ether.h | 2 + lib/librte_net/rte_mpls.h | 42 + lib/librte_net/rte_net.c | 21 +- lib/librte_net/rte_net.h | 20 +- lib/librte_pdump/Makefile | 2 - lib/librte_pipeline/Makefile | 2 +- lib/librte_pipeline/meson.build | 2 +- lib/librte_pipeline/rte_pipeline.c | 3 +- lib/librte_pipeline/rte_pipeline_version.map | 1 + lib/librte_pipeline/rte_table_action.c | 1048 +- lib/librte_pipeline/rte_table_action.h | 203 + lib/librte_port/Makefile | 4 +- lib/librte_port/meson.build | 8 +- lib/librte_port/rte_port_sym_crypto.c | 552 + lib/librte_port/rte_port_sym_crypto.h | 93 + lib/librte_port/rte_port_version.map | 9 + lib/librte_power/Makefile | 5 +- lib/librte_power/channel_commands.h | 5 + lib/librte_power/meson.build | 6 +- lib/librte_power/rte_power_empty_poll.c | 545 + lib/librte_power/rte_power_empty_poll.h | 223 + lib/librte_power/rte_power_version.map | 13 + lib/librte_rawdev/rte_rawdev.c | 10 +- lib/librte_rawdev/rte_rawdev_pmd.h | 2 - lib/librte_ring/meson.build | 1 + lib/librte_ring/rte_ring.h | 4 +- lib/librte_sched/Makefile | 2 - lib/librte_sched/rte_sched.c | 5 +- lib/librte_security/rte_security.c | 4 + lib/librte_security/rte_security.h | 90 + lib/librte_table/Makefile | 2 + lib/librte_table/meson.build | 2 + lib/librte_table/rte_table_hash_func.h | 245 + lib/librte_table/rte_table_hash_func_arm64.h | 21 + lib/librte_telemetry/Makefile | 30 + lib/librte_telemetry/meson.build | 15 + lib/librte_telemetry/rte_telemetry.c | 1813 ++ lib/librte_telemetry/rte_telemetry.h | 66 + lib/librte_telemetry/rte_telemetry_internal.h | 81 + lib/librte_telemetry/rte_telemetry_parser.c | 586 + lib/librte_telemetry/rte_telemetry_parser.h | 14 + lib/librte_telemetry/rte_telemetry_parser_test.c | 534 + lib/librte_telemetry/rte_telemetry_parser_test.h | 39 + lib/librte_telemetry/rte_telemetry_socket_tests.h | 36 + lib/librte_telemetry/rte_telemetry_version.map | 10 + lib/librte_vhost/Makefile | 5 +- lib/librte_vhost/meson.build | 3 + lib/librte_vhost/rte_vdpa.h | 97 +- lib/librte_vhost/rte_vhost.h | 5 + lib/librte_vhost/rte_vhost_version.map | 1 + lib/librte_vhost/socket.c | 46 +- lib/librte_vhost/vdpa.c | 6 + lib/librte_vhost/vhost.c | 26 +- lib/librte_vhost/vhost.h | 34 +- lib/librte_vhost/vhost_crypto.c | 25 +- lib/librte_vhost/vhost_user.c | 683 +- lib/librte_vhost/vhost_user.h | 12 +- lib/librte_vhost/virtio_net.c | 52 +- lib/meson.build | 31 +- meson.build | 54 +- meson_options.txt | 8 +- mk/internal/rte.compile-pre.mk | 2 +- mk/rte.app.mk | 31 +- mk/rte.cpuflags.mk | 2 +- mk/rte.sdkdoc.mk | 21 +- mk/target/generic/rte.vars.mk | 3 + mk/toolchain/gcc/rte.toolchain-compat.mk | 4 +- test/test-pipeline/init.c | 2 +- test/test/Makefile | 8 +- test/test/autotest_data.py | 40 +- test/test/meson.build | 24 +- test/test/test.c | 33 +- test/test/test_acl.h | 18 +- test/test/test_alarm.c | 5 +- test/test/test_bitmap.c | 18 + test/test/test_cryptodev.c | 714 +- test/test/test_cryptodev.h | 2 + test/test/test_cryptodev_aes_test_vectors.h | 147 +- test/test/test_cryptodev_blockcipher.c | 16 +- test/test/test_cryptodev_blockcipher.h | 2 + test/test/test_cryptodev_des_test_vectors.h | 26 +- test/test/test_cryptodev_hash_test_vectors.h | 104 +- test/test/test_cycles.c | 51 +- test/test/test_event_eth_rx_adapter.c | 70 +- test/test/test_event_eth_tx_adapter.c | 699 + test/test/test_event_timer_adapter.c | 2 +- test/test/test_eventdev.c | 21 +- test/test/test_external_mem.c | 390 + test/test/test_hash.c | 299 +- test/test/test_hash_multiwriter.c | 3 +- test/test/test_hash_perf.c | 127 +- test/test/test_hash_readwrite.c | 156 +- test/test/test_hash_readwrite_lf.c | 1220 + test/test/test_hash_scaling.c | 191 - test/test/test_kni.c | 135 +- test/test/test_kvargs.c | 21 + test/test/test_link_bonding_mode4.c | 1 - test/test/test_malloc.c | 3 + test/test/test_memzone.c | 3 + test/test/test_metrics.c | 313 + test/test/test_pmd_perf.c | 1 - test/test/test_pmd_ring.c | 326 +- test/test/test_timer_racecond.c | 2 +- test/test/virtual_pmd.c | 1 - usertools/dpdk-telemetry-client.py | 116 + 1190 files changed, 127125 insertions(+), 38438 deletions(-) create mode 100644 app/test-pmd/noisy_vnf.c create mode 100644 app/test-pmd/util.c create mode 100644 config/ppc_64/meson.build create mode 100755 devtools/check-forbidden-tokens.awk delete mode 100644 doc/api/doxy-api.conf create mode 100644 doc/api/doxy-api.conf.in create mode 100755 doc/api/generate_doxygen.sh create mode 100755 doc/api/generate_examples.sh create mode 100644 doc/api/meson.build create mode 100644 doc/guides/cryptodevs/caam_jr.rst create mode 100644 doc/guides/cryptodevs/features/caam_jr.ini create mode 100644 doc/guides/cryptodevs/features/octeontx.ini create mode 100644 doc/guides/cryptodevs/octeontx.rst create mode 100644 doc/guides/eventdevs/dsw.rst create mode 100644 doc/guides/howto/telemetry.rst create mode 100644 doc/guides/meson.build create mode 100644 doc/guides/nics/atlantic.rst create mode 100644 doc/guides/nics/enetc.rst create mode 100644 doc/guides/nics/features/atlantic.ini create mode 100644 doc/guides/nics/features/enetc.ini create mode 100644 doc/guides/nics/features/mvneta.ini create mode 100644 doc/guides/nics/img/mvpp2_tm.svg create mode 100644 doc/guides/nics/mvneta.rst create mode 100644 doc/guides/prog_guide/event_ethernet_tx_adapter.rst delete mode 100644 doc/guides/prog_guide/port_hotplug_framework.rst delete mode 100644 doc/guides/rel_notes/rel_description.rst create mode 100644 doc/guides/rel_notes/release_18_11.rst create mode 100644 doc/guides/sample_app_ug/vdpa.rst rename doc/guides/tools/img/{eventdev_pipeline_atq_test_lockfree.svg => eventdev_pipeline_atq_test_internal_port.svg} (99%) rename doc/guides/tools/img/{eventdev_pipeline_queue_test_lockfree.svg => eventdev_pipeline_queue_test_internal_port.svg} (99%) create mode 100644 doc/meson.build create mode 100644 drivers/baseband/meson.build create mode 100644 drivers/baseband/null/meson.build create mode 100644 drivers/bus/fslmc/mc/fsl_dpopr.h create mode 100644 drivers/bus/pci/pci_params.c create mode 100644 drivers/bus/vdev/vdev_params.c create mode 100644 drivers/bus/vdev/vdev_private.h create mode 100644 drivers/common/cpt/Makefile create mode 100644 drivers/common/cpt/cpt_common.h create mode 100644 drivers/common/cpt/cpt_hw_types.h create mode 100644 drivers/common/cpt/cpt_mcode_defines.h create mode 100644 drivers/common/cpt/cpt_pmd_logs.h create mode 100644 drivers/common/cpt/cpt_pmd_ops_helper.c create mode 100644 drivers/common/cpt/cpt_pmd_ops_helper.h create mode 100644 drivers/common/cpt/cpt_request_mgr.h create mode 100644 drivers/common/cpt/cpt_ucode.h create mode 100644 drivers/common/cpt/meson.build create mode 100644 drivers/common/cpt/rte_common_cpt_version.map create mode 100644 drivers/common/dpaax/Makefile create mode 100644 drivers/common/dpaax/dpaax_iova_table.c create mode 100644 drivers/common/dpaax/dpaax_iova_table.h create mode 100644 drivers/common/dpaax/dpaax_logs.h create mode 100644 drivers/common/dpaax/meson.build create mode 100644 drivers/common/dpaax/rte_common_dpaax_version.map create mode 100644 drivers/common/mvep/Makefile create mode 100644 drivers/common/mvep/meson.build create mode 100644 drivers/common/mvep/mvep_common.c create mode 100644 drivers/common/mvep/rte_common_mvep_version.map create mode 100644 drivers/common/mvep/rte_mvep_common.h create mode 100644 drivers/crypto/aesni_gcm/meson.build create mode 100644 drivers/crypto/aesni_mb/meson.build create mode 100644 drivers/crypto/caam_jr/Makefile create mode 100644 drivers/crypto/caam_jr/caam_jr.c create mode 100644 drivers/crypto/caam_jr/caam_jr_capabilities.c create mode 100644 drivers/crypto/caam_jr/caam_jr_capabilities.h create mode 100644 drivers/crypto/caam_jr/caam_jr_config.h create mode 100644 drivers/crypto/caam_jr/caam_jr_desc.h create mode 100644 drivers/crypto/caam_jr/caam_jr_hw.c create mode 100644 drivers/crypto/caam_jr/caam_jr_hw_specific.h create mode 100644 drivers/crypto/caam_jr/caam_jr_log.h create mode 100644 drivers/crypto/caam_jr/caam_jr_pvt.h create mode 100644 drivers/crypto/caam_jr/caam_jr_uio.c create mode 100644 drivers/crypto/caam_jr/meson.build create mode 100644 drivers/crypto/caam_jr/rte_pmd_caam_jr_version.map create mode 100644 drivers/crypto/dpaa2_sec/dpaa2_sec_event.h create mode 100644 drivers/crypto/dpaa2_sec/hw/desc/pdcp.h create mode 100644 drivers/crypto/kasumi/meson.build create mode 100644 drivers/crypto/octeontx/Makefile create mode 100644 drivers/crypto/octeontx/meson.build create mode 100644 drivers/crypto/octeontx/otx_cryptodev.c create mode 100644 drivers/crypto/octeontx/otx_cryptodev.h create mode 100644 drivers/crypto/octeontx/otx_cryptodev_capabilities.c create mode 100644 drivers/crypto/octeontx/otx_cryptodev_capabilities.h create mode 100644 drivers/crypto/octeontx/otx_cryptodev_hw_access.c create mode 100644 drivers/crypto/octeontx/otx_cryptodev_hw_access.h create mode 100644 drivers/crypto/octeontx/otx_cryptodev_mbox.c create mode 100644 drivers/crypto/octeontx/otx_cryptodev_mbox.h create mode 100644 drivers/crypto/octeontx/otx_cryptodev_ops.c create mode 100644 drivers/crypto/octeontx/otx_cryptodev_ops.h create mode 100644 drivers/crypto/octeontx/rte_pmd_octeontx_crypto_version.map create mode 100644 drivers/crypto/scheduler/meson.build create mode 100644 drivers/crypto/zuc/meson.build create mode 100644 drivers/event/dsw/Makefile create mode 100644 drivers/event/dsw/dsw_evdev.c create mode 100644 drivers/event/dsw/dsw_evdev.h create mode 100644 drivers/event/dsw/dsw_event.c create mode 100644 drivers/event/dsw/dsw_sort.h create mode 100644 drivers/event/dsw/dsw_xstats.c create mode 100644 drivers/event/dsw/meson.build create mode 100644 drivers/event/dsw/rte_pmd_dsw_event_version.map create mode 100644 drivers/event/opdl/meson.build rename drivers/event/opdl/{rte_pmd_evdev_opdl_version.map => rte_pmd_opdl_event_version.map} (100%) create mode 100644 drivers/net/atlantic/Makefile create mode 100644 drivers/net/atlantic/atl_common.h create mode 100644 drivers/net/atlantic/atl_ethdev.c create mode 100644 drivers/net/atlantic/atl_ethdev.h create mode 100644 drivers/net/atlantic/atl_hw_regs.c create mode 100644 drivers/net/atlantic/atl_hw_regs.h create mode 100644 drivers/net/atlantic/atl_logs.h create mode 100644 drivers/net/atlantic/atl_rxtx.c create mode 100644 drivers/net/atlantic/atl_types.h create mode 100644 drivers/net/atlantic/hw_atl/hw_atl_b0.c create mode 100644 drivers/net/atlantic/hw_atl/hw_atl_b0.h create mode 100644 drivers/net/atlantic/hw_atl/hw_atl_b0_internal.h create mode 100644 drivers/net/atlantic/hw_atl/hw_atl_llh.c create mode 100644 drivers/net/atlantic/hw_atl/hw_atl_llh.h create mode 100644 drivers/net/atlantic/hw_atl/hw_atl_llh_internal.h create mode 100644 drivers/net/atlantic/hw_atl/hw_atl_utils.c create mode 100644 drivers/net/atlantic/hw_atl/hw_atl_utils.h create mode 100644 drivers/net/atlantic/hw_atl/hw_atl_utils_fw2x.c create mode 100644 drivers/net/atlantic/meson.build create mode 100644 drivers/net/atlantic/rte_pmd_atlantic_version.map create mode 100644 drivers/net/avf/base/meson.build create mode 100644 drivers/net/avf/meson.build create mode 100644 drivers/net/cxgbe/l2t.c create mode 100644 drivers/net/cxgbe/l2t.h create mode 100644 drivers/net/cxgbe/mps_tcam.c create mode 100644 drivers/net/cxgbe/mps_tcam.h create mode 100644 drivers/net/enetc/Makefile create mode 100644 drivers/net/enetc/base/enetc_hw.h create mode 100644 drivers/net/enetc/enetc.h create mode 100644 drivers/net/enetc/enetc_ethdev.c create mode 100644 drivers/net/enetc/enetc_logs.h create mode 100644 drivers/net/enetc/enetc_rxtx.c create mode 100644 drivers/net/enetc/meson.build create mode 100644 drivers/net/enetc/rte_pmd_enetc_version.map create mode 100644 drivers/net/enic/enic_rxtx_common.h create mode 100644 drivers/net/enic/enic_rxtx_vec_avx2.c create mode 100644 drivers/net/mlx4/meson.build create mode 100644 drivers/net/mlx5/meson.build create mode 100644 drivers/net/mlx5/mlx5_flow.h create mode 100644 drivers/net/mlx5/mlx5_flow_dv.c create mode 100644 drivers/net/mlx5/mlx5_flow_tcf.c create mode 100644 drivers/net/mlx5/mlx5_flow_verbs.c delete mode 100644 drivers/net/mlx5/mlx5_nl_flow.c create mode 100644 drivers/net/mvneta/Makefile create mode 100644 drivers/net/mvneta/meson.build create mode 100644 drivers/net/mvneta/mvneta_ethdev.c create mode 100644 drivers/net/mvneta/mvneta_ethdev.h create mode 100644 drivers/net/mvneta/mvneta_rxtx.c create mode 100644 drivers/net/mvneta/mvneta_rxtx.h create mode 100644 drivers/net/mvneta/rte_pmd_mvneta_version.map create mode 100644 drivers/net/mvpp2/mrvl_flow.h create mode 100644 drivers/net/mvpp2/mrvl_mtr.c create mode 100644 drivers/net/mvpp2/mrvl_mtr.h create mode 100644 drivers/net/mvpp2/mrvl_tm.c create mode 100644 drivers/net/mvpp2/mrvl_tm.h create mode 100644 drivers/net/netvsc/hn_vf.c create mode 100644 drivers/net/qede/base/meson.build create mode 100644 drivers/net/qede/meson.build delete mode 100644 drivers/net/qede/qede_fdir.c create mode 100644 drivers/net/qede/qede_filter.c create mode 100644 drivers/net/sfc/base/efx_annote.h create mode 100644 drivers/net/sfc/base/mc_driver_pcol_strs.h create mode 100644 drivers/net/sfc/sfc_tso.h delete mode 100644 drivers/net/softnic/hash_func.h delete mode 100644 drivers/net/softnic/hash_func_arm64.h create mode 100644 drivers/net/softnic/rte_eth_softnic_cryptodev.c create mode 100644 drivers/net/softnic/rte_eth_softnic_flow.c create mode 100644 drivers/net/softnic/rte_eth_softnic_meter.c create mode 100644 drivers/net/tap/meson.build create mode 100644 drivers/net/vdev_netvsc/meson.build create mode 100644 drivers/net/vmxnet3/meson.build create mode 100644 examples/ip_pipeline/cryptodev.c create mode 100644 examples/ip_pipeline/cryptodev.h create mode 100644 examples/ip_pipeline/examples/flow_crypto.cli delete mode 100644 examples/ip_pipeline/hash_func.h delete mode 100644 examples/ip_pipeline/hash_func_arm64.h create mode 100644 examples/multi_process/hotplug_mp/Makefile create mode 100644 examples/multi_process/hotplug_mp/commands.c create mode 100644 examples/multi_process/hotplug_mp/commands.h create mode 100644 examples/multi_process/hotplug_mp/main.c create mode 100644 examples/vdpa/Makefile create mode 100644 examples/vdpa/main.c create mode 100644 examples/vdpa/meson.build create mode 100644 examples/vm_power_manager/guest_cli/meson.build create mode 100644 kernel/linux/kni/Kbuild create mode 100644 kernel/linux/kni/ethtool/igb/meson.build create mode 100644 kernel/linux/kni/ethtool/ixgbe/meson.build create mode 100644 kernel/linux/kni/ethtool/meson.build create mode 100644 kernel/linux/kni/meson.build create mode 100644 lib/librte_eal/common/arch/ppc_64/meson.build create mode 100644 lib/librte_eal/common/hotplug_mp.c create mode 100644 lib/librte_eal/common/hotplug_mp.h create mode 100644 lib/librte_eal/common/include/arch/ppc_64/meson.build create mode 100644 lib/librte_eal/common/include/rte_option.h create mode 100644 lib/librte_eal/common/rte_option.c create mode 100644 lib/librte_ethdev/ethdev_private.c create mode 100644 lib/librte_ethdev/ethdev_private.h create mode 100644 lib/librte_ethdev/rte_class_eth.c create mode 100644 lib/librte_eventdev/rte_event_eth_tx_adapter.c create mode 100644 lib/librte_eventdev/rte_event_eth_tx_adapter.h create mode 100644 lib/librte_net/rte_mpls.h create mode 100644 lib/librte_port/rte_port_sym_crypto.c create mode 100644 lib/librte_port/rte_port_sym_crypto.h create mode 100644 lib/librte_power/rte_power_empty_poll.c create mode 100644 lib/librte_power/rte_power_empty_poll.h create mode 100644 lib/librte_table/rte_table_hash_func.h create mode 100644 lib/librte_table/rte_table_hash_func_arm64.h create mode 100644 lib/librte_telemetry/Makefile create mode 100644 lib/librte_telemetry/meson.build create mode 100644 lib/librte_telemetry/rte_telemetry.c create mode 100644 lib/librte_telemetry/rte_telemetry.h create mode 100644 lib/librte_telemetry/rte_telemetry_internal.h create mode 100644 lib/librte_telemetry/rte_telemetry_parser.c create mode 100644 lib/librte_telemetry/rte_telemetry_parser.h create mode 100644 lib/librte_telemetry/rte_telemetry_parser_test.c create mode 100644 lib/librte_telemetry/rte_telemetry_parser_test.h create mode 100644 lib/librte_telemetry/rte_telemetry_socket_tests.h create mode 100644 lib/librte_telemetry/rte_telemetry_version.map create mode 100644 test/test/test_event_eth_tx_adapter.c create mode 100644 test/test/test_external_mem.c create mode 100644 test/test/test_hash_readwrite_lf.c delete mode 100644 test/test/test_hash_scaling.c create mode 100644 test/test/test_metrics.c create mode 100644 usertools/dpdk-telemetry-client.py diff --git a/MAINTAINERS b/MAINTAINERS index 9fd258fa..e60379df 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -81,6 +81,7 @@ M: Thomas Monjalon F: MAINTAINERS F: devtools/check-dup-includes.sh F: devtools/check-maintainers.sh +F: devtools/check-forbidden-tokens.awk F: devtools/check-git-log.sh F: devtools/check-includes.sh F: devtools/check-symbol-maps.sh @@ -177,6 +178,7 @@ F: lib/librte_eal/common/eal_hugepages.h F: lib/librte_eal/linuxapp/eal/eal_mem* F: lib/librte_eal/bsdapp/eal/eal_mem* F: doc/guides/prog_guide/env_abstraction_layer.rst +F: test/test/test_external_mem.c F: test/test/test_func_reentrancy.c F: test/test/test_fbarray.c F: test/test/test_malloc.c @@ -377,6 +379,13 @@ F: lib/librte_eventdev/*eth_rx_adapter* F: test/test/test_event_eth_rx_adapter.c F: doc/guides/prog_guide/event_ethernet_rx_adapter.rst +Eventdev Ethdev Tx Adapter API - EXPERIMENTAL +M: Nikhil Rao +T: git://dpdk.org/next/dpdk-next-eventdev +F: lib/librte_eventdev/*eth_tx_adapter* +F: test/test/test_event_eth_tx_adapter.c +F: doc/guides/prog_guide/event_ethernet_tx_adapter.rst + Eventdev Timer Adapter API - EXPERIMENTAL M: Erik Gabriel Carrillo T: git://dpdk.org/next/dpdk-next-eventdev @@ -419,6 +428,7 @@ F: drivers/bus/ifpga/ NXP buses M: Hemant Agrawal M: Shreyansh Jain +F: drivers/common/dpaax/ F: drivers/bus/dpaa/ F: drivers/bus/fslmc/ @@ -477,6 +487,13 @@ F: drivers/net/axgbe/ F: doc/guides/nics/axgbe.rst F: doc/guides/nics/features/axgbe.ini +Aquantia atlantic +M: Igor Russkikh +M: Pavel Belous +F: drivers/net/atlantic/ +F: doc/guides/nics/atlantic.rst +F: doc/guides/nics/features/atlantic.ini + Atomic Rules ARK M: Shepard Siegel M: Ed Czeck @@ -506,8 +523,7 @@ F: drivers/net/liquidio/ F: doc/guides/nics/liquidio.rst F: doc/guides/nics/features/liquidio.ini -Cavium OcteonTX -M: Santosh Shukla +Cavium OCTEON TX M: Jerin Jacob F: drivers/common/octeontx/ F: drivers/mempool/octeontx/ @@ -581,10 +597,19 @@ Marvell mvpp2 M: Tomasz Duszynski M: Dmitri Epshtein M: Natalie Samsonov +F: drivers/common/mvep/ F: drivers/net/mvpp2/ F: doc/guides/nics/mvpp2.rst F: doc/guides/nics/features/mvpp2.ini +Marvell mvneta +M: Zyta Szpak +M: Dmitri Epshtein +M: Natalie Samsonov +F: drivers/net/mvneta/ +F: doc/guides/nics/mvneta.rst +F: doc/guides/nics/features/mvneta.ini + Mellanox mlx4 M: Matan Azrad M: Shahaf Shuler @@ -643,6 +668,13 @@ F: drivers/net/dpaa2/ F: doc/guides/nics/dpaa2.rst F: doc/guides/nics/features/dpaa2.ini +NXP enetc +M: Gagandeep Singh +M: Pankaj Chauhan +F: drivers/net/enetc/ +F: doc/guides/nics/enetc.rst +F: doc/guides/nics/features/enetc.ini + QLogic bnx2x M: Harish Patil M: Rasesh Mody @@ -682,6 +714,8 @@ F: doc/guides/sample_app_ug/vhost.rst F: examples/vhost_scsi/ F: doc/guides/sample_app_ug/vhost_scsi.rst F: examples/vhost_crypto/ +F: examples/vdpa/ +F: doc/guides/sample_app_ug/vdpa.rst Vhost PMD M: Maxime Coquelin @@ -771,6 +805,13 @@ F: drivers/crypto/armv8/ F: doc/guides/cryptodevs/armv8.rst F: doc/guides/cryptodevs/features/armv8.ini +Cavium OCTEON TX crypto +M: Anoob Joseph +F: drivers/common/cpt/ +F: drivers/crypto/octeontx/ +F: doc/guides/cryptodevs/octeontx.rst +F: doc/guides/cryptodevs/features/octeontx.ini + Crypto Scheduler M: Fan Zhang F: drivers/crypto/scheduler/ @@ -817,6 +858,13 @@ F: drivers/crypto/null/ F: doc/guides/cryptodevs/null.rst F: doc/guides/cryptodevs/features/null.ini +NXP CAAM JR +M: Gagandeep Singh +M: Hemant Agrawal +F: drivers/crypto/caam_jr/ +F: doc/guides/cryptodevs/caam_jr.rst +F: doc/guides/cryptodevs/features/caam_jr.ini + NXP DPAA_SEC M: Akhil Goyal M: Hemant Agrawal @@ -861,7 +909,7 @@ Compression Drivers M: Pablo de Lara T: git://dpdk.org/next/dpdk-next-crypto -Cavium OCTEONTX zipvf +Cavium OCTEON TX zipvf M: Ashish Gupta F: drivers/compress/octeontx/ F: doc/guides/compressdevs/octeontx.rst @@ -890,13 +938,12 @@ Eventdev Drivers M: Jerin Jacob T: git://dpdk.org/next/dpdk-next-eventdev -Cavium OCTEONTX ssovf +Cavium OCTEON TX ssovf M: Jerin Jacob -M: Santosh Shukla F: drivers/event/octeontx/ F: doc/guides/eventdevs/octeontx.rst -Cavium OCTEONTX timvf +Cavium OCTEON TX timvf M: Pavan Nikhilesh F: drivers/event/octeontx/timvf_* @@ -919,6 +966,11 @@ F: doc/guides/eventdevs/sw.rst F: examples/eventdev_pipeline/ F: doc/guides/sample_app_ug/eventdev_pipeline.rst +Distributed Software Eventdev PMD +M: Mattias Rönnblom +F: drivers/event/dsw/ +F: doc/guides/eventdevs/dsw.rst + Software OPDL Eventdev PMD M: Liang Ma M: Peter Mccarthy @@ -1140,6 +1192,7 @@ F: doc/guides/sample_app_ug/l2_forward_job_stats.rst Metrics M: Remy Horton F: lib/librte_metrics/ +F: test/test/test_metrics.c Bit-rate statistics M: Remy Horton @@ -1149,6 +1202,12 @@ Latency statistics M: Reshma Pattan F: lib/librte_latencystats/ +Telemetry - EXPERIMENTAL +M: Kevin Laatz +F: lib/librte_telemetry/ +F: usertools/dpdk-telemetry-client.py +F: doc/guides/howto/telemetry.rst + BPF - EXPERIMENTAL M: Konstantin Ananyev F: lib/librte_bpf/ diff --git a/app/meson.build b/app/meson.build index 99e0b93e..a9a026bb 100644 --- a/app/meson.build +++ b/app/meson.build @@ -11,20 +11,25 @@ apps = ['pdump', # for BSD only lib_execinfo = cc.find_library('execinfo', required: false) +default_cflags = machine_args + +# specify -D_GNU_SOURCE unconditionally +default_cflags += '-D_GNU_SOURCE' + foreach app:apps build = true name = app allow_experimental_apis = false sources = [] includes = [] - cflags = machine_args + cflags = default_cflags objs = [] # other object files to link against, used e.g. for # instruction-set optimized versions of code # use "deps" for internal DPDK dependencies, and "ext_deps" for # external package/library requirements ext_deps = [] - deps = [] + deps = dpdk_app_link_libraries subdir(name) @@ -38,7 +43,7 @@ foreach app:apps link_libs = [] if get_option('default_library') == 'static' - link_libs = dpdk_drivers + link_libs = dpdk_static_libraries + dpdk_drivers endif if allow_experimental_apis diff --git a/app/pdump/main.c b/app/pdump/main.c index ac228712..d96556ec 100644 --- a/app/pdump/main.c +++ b/app/pdump/main.c @@ -81,7 +81,7 @@ enum pdump_by { DEVICE_ID = 2 }; -const char *valid_pdump_arguments[] = { +static const char * const valid_pdump_arguments[] = { PDUMP_PORT_ARG, PDUMP_PCI_ARG, PDUMP_QUEUE_ARG, @@ -136,9 +136,9 @@ struct parse_val { uint64_t val; }; -int num_tuples; +static int num_tuples; static struct rte_eth_conf port_conf_default; -volatile uint8_t quit_signal; +static volatile uint8_t quit_signal; /**< display usage */ static void diff --git a/app/pdump/meson.build b/app/pdump/meson.build index 988cb4eb..116c27f0 100644 --- a/app/pdump/meson.build +++ b/app/pdump/meson.build @@ -3,4 +3,4 @@ sources = files('main.c') allow_experimental_apis = true -deps = ['ethdev', 'kvargs', 'pdump'] +deps += ['ethdev', 'kvargs', 'pdump'] diff --git a/app/proc-info/meson.build b/app/proc-info/meson.build index 9c148e36..a52b2ee4 100644 --- a/app/proc-info/meson.build +++ b/app/proc-info/meson.build @@ -3,4 +3,4 @@ sources = files('main.c') allow_experimental_apis = true -deps = ['ethdev', 'metrics'] +deps += ['ethdev', 'metrics'] diff --git a/app/test-bbdev/meson.build b/app/test-bbdev/meson.build index 653907de..eb8cc049 100644 --- a/app/test-bbdev/meson.build +++ b/app/test-bbdev/meson.build @@ -6,4 +6,4 @@ sources = files('main.c', 'test_bbdev_perf.c', 'test_bbdev_vector.c') allow_experimental_apis = true -deps = ['bbdev', 'bus_vdev'] +deps += ['bbdev', 'bus_vdev'] diff --git a/app/test-bbdev/test_bbdev_perf.c b/app/test-bbdev/test_bbdev_perf.c index 6861edc4..fbe6cc91 100644 --- a/app/test-bbdev/test_bbdev_perf.c +++ b/app/test-bbdev/test_bbdev_perf.c @@ -267,12 +267,13 @@ create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id, static int create_mempools(struct active_device *ad, int socket_id, - enum rte_bbdev_op_type op_type, uint16_t num_ops) + enum rte_bbdev_op_type org_op_type, uint16_t num_ops) { struct rte_mempool *mp; unsigned int ops_pool_size, mbuf_pool_size = 0; char pool_name[RTE_MEMPOOL_NAMESIZE]; const char *op_type_str; + enum rte_bbdev_op_type op_type = org_op_type; struct op_data_entries *in = &test_vector.entries[DATA_INPUT]; struct op_data_entries *hard_out = @@ -289,6 +290,9 @@ create_mempools(struct active_device *ad, int socket_id, OPS_CACHE_SIZE + 1)), OPS_POOL_SIZE_MIN)); + if (org_op_type == RTE_BBDEV_OP_NONE) + op_type = RTE_BBDEV_OP_TURBO_ENC; + op_type_str = rte_bbdev_op_type_str(op_type); TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); @@ -303,6 +307,10 @@ create_mempools(struct active_device *ad, int socket_id, socket_id); ad->ops_mempool = mp; + /* Do not create inputs and outputs mbufs for BaseBand Null Device */ + if (org_op_type == RTE_BBDEV_OP_NONE) + return TEST_SUCCESS; + /* Inputs */ mbuf_pool_size = optimal_mempool_size(ops_pool_size * in->nb_segments); mp = create_mbuf_pool(in, ad->dev_id, socket_id, mbuf_pool_size, "in"); @@ -1058,14 +1066,14 @@ run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id, rte_bbdev_info_get(ad->dev_id, &info); socket_id = GET_SOCKET(info.socket_id); - if (op_type == RTE_BBDEV_OP_NONE) - op_type = RTE_BBDEV_OP_TURBO_ENC; f_ret = create_mempools(ad, socket_id, op_type, get_num_ops()); if (f_ret != TEST_SUCCESS) { printf("Couldn't create mempools"); goto fail; } + if (op_type == RTE_BBDEV_OP_NONE) + op_type = RTE_BBDEV_OP_TURBO_ENC; f_ret = init_test_op_params(op_params, test_vector.op_type, test_vector.expected_status, diff --git a/app/test-crypto-perf/cperf_test_vectors.c b/app/test-crypto-perf/cperf_test_vectors.c index 907a995c..1af95249 100644 --- a/app/test-crypto-perf/cperf_test_vectors.c +++ b/app/test-crypto-perf/cperf_test_vectors.c @@ -419,13 +419,19 @@ cperf_test_vector_get_dummy(struct cperf_options *options) t_vec->cipher_key.length = 0; t_vec->ciphertext.data = plaintext; t_vec->cipher_key.data = NULL; - t_vec->cipher_iv.data = NULL; } else { t_vec->cipher_key.length = options->cipher_key_sz; t_vec->ciphertext.data = ciphertext; t_vec->cipher_key.data = cipher_key; - t_vec->cipher_iv.data = rte_malloc(NULL, options->cipher_iv_sz, - 16); + } + + /* Init IV data ptr */ + t_vec->cipher_iv.data = NULL; + + if (options->cipher_iv_sz != 0) { + /* Set IV parameters */ + t_vec->cipher_iv.data = rte_malloc(NULL, + options->cipher_iv_sz, 16); if (t_vec->cipher_iv.data == NULL) { rte_free(t_vec); return NULL; @@ -433,17 +439,7 @@ cperf_test_vector_get_dummy(struct cperf_options *options) memcpy(t_vec->cipher_iv.data, iv, options->cipher_iv_sz); } t_vec->ciphertext.length = options->max_buffer_size; - - /* Set IV parameters */ - t_vec->cipher_iv.data = rte_malloc(NULL, options->cipher_iv_sz, - 16); - if (options->cipher_iv_sz && t_vec->cipher_iv.data == NULL) { - rte_free(t_vec); - return NULL; - } - memcpy(t_vec->cipher_iv.data, iv, options->cipher_iv_sz); t_vec->cipher_iv.length = options->cipher_iv_sz; - t_vec->data.cipher_offset = 0; t_vec->data.cipher_length = options->max_buffer_size; diff --git a/app/test-crypto-perf/main.c b/app/test-crypto-perf/main.c index 5c7dadb6..953e058c 100644 --- a/app/test-crypto-perf/main.c +++ b/app/test-crypto-perf/main.c @@ -342,7 +342,9 @@ cperf_check_test_vector(struct cperf_options *opts, return -1; if (test_vec->ciphertext.length < opts->max_buffer_size) return -1; - if (test_vec->cipher_iv.data == NULL) + /* Cipher IV is only required for some algorithms */ + if (opts->cipher_iv_sz && + test_vec->cipher_iv.data == NULL) return -1; if (test_vec->cipher_iv.length != opts->cipher_iv_sz) return -1; @@ -357,7 +359,9 @@ cperf_check_test_vector(struct cperf_options *opts, return -1; if (test_vec->plaintext.length < opts->max_buffer_size) return -1; - if (test_vec->auth_key.data == NULL) + /* Auth key is only required for some algorithms */ + if (opts->auth_key_sz && + test_vec->auth_key.data == NULL) return -1; if (test_vec->auth_key.length != opts->auth_key_sz) return -1; @@ -421,6 +425,10 @@ cperf_check_test_vector(struct cperf_options *opts, return -1; if (test_vec->ciphertext.length < opts->max_buffer_size) return -1; + if (test_vec->aead_key.data == NULL) + return -1; + if (test_vec->aead_key.length != opts->aead_key_sz) + return -1; if (test_vec->aead_iv.data == NULL) return -1; if (test_vec->aead_iv.length != opts->aead_iv_sz) diff --git a/app/test-crypto-perf/meson.build b/app/test-crypto-perf/meson.build index eacd7a0f..d735b186 100644 --- a/app/test-crypto-perf/meson.build +++ b/app/test-crypto-perf/meson.build @@ -12,4 +12,4 @@ sources = files('cperf_ops.c', 'cperf_test_vectors.c', 'cperf_test_verify.c', 'main.c') -deps = ['cryptodev'] +deps += ['cryptodev'] diff --git a/app/test-eventdev/test_perf_common.c b/app/test-eventdev/test_perf_common.c index d33cb2cd..86187753 100644 --- a/app/test-eventdev/test_perf_common.c +++ b/app/test-eventdev/test_perf_common.c @@ -680,7 +680,6 @@ perf_ethdev_setup(struct evt_test *test, struct evt_options *opt) .mq_mode = ETH_MQ_RX_RSS, .max_rx_pkt_len = ETHER_MAX_LEN, .split_hdr_size = 0, - .offloads = DEV_RX_OFFLOAD_CRC_STRIP, }, .rx_adv_conf = { .rss_conf = { diff --git a/app/test-eventdev/test_pipeline_atq.c b/app/test-eventdev/test_pipeline_atq.c index 26dc79f9..c60635bf 100644 --- a/app/test-eventdev/test_pipeline_atq.c +++ b/app/test-eventdev/test_pipeline_atq.c @@ -15,10 +15,10 @@ pipeline_atq_nb_event_queues(struct evt_options *opt) return rte_eth_dev_count_avail(); } -static int +static __rte_noinline int pipeline_atq_worker_single_stage_tx(void *arg) { - PIPELINE_WROKER_SINGLE_STAGE_INIT; + PIPELINE_WORKER_SINGLE_STAGE_INIT; while (t->done == false) { uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0); @@ -28,23 +28,18 @@ pipeline_atq_worker_single_stage_tx(void *arg) continue; } - if (ev.sched_type == RTE_SCHED_TYPE_ATOMIC) { - pipeline_tx_pkt(ev.mbuf); - w->processed_pkts++; - continue; - } - pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC); - pipeline_event_enqueue(dev, port, &ev); + pipeline_event_tx(dev, port, &ev); + w->processed_pkts++; } return 0; } -static int +static __rte_noinline int pipeline_atq_worker_single_stage_fwd(void *arg) { - PIPELINE_WROKER_SINGLE_STAGE_INIT; - const uint8_t tx_queue = t->tx_service.queue_id; + PIPELINE_WORKER_SINGLE_STAGE_INIT; + const uint8_t *tx_queue = t->tx_evqueue_id; while (t->done == false) { uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0); @@ -54,19 +49,19 @@ pipeline_atq_worker_single_stage_fwd(void *arg) continue; } - w->processed_pkts++; - ev.queue_id = tx_queue; + ev.queue_id = tx_queue[ev.mbuf->port]; pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC); pipeline_event_enqueue(dev, port, &ev); + w->processed_pkts++; } return 0; } -static int +static __rte_noinline int pipeline_atq_worker_single_stage_burst_tx(void *arg) { - PIPELINE_WROKER_SINGLE_STAGE_BURST_INIT; + PIPELINE_WORKER_SINGLE_STAGE_BURST_INIT; while (t->done == false) { uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev, @@ -79,27 +74,21 @@ pipeline_atq_worker_single_stage_burst_tx(void *arg) for (i = 0; i < nb_rx; i++) { rte_prefetch0(ev[i + 1].mbuf); - if (ev[i].sched_type == RTE_SCHED_TYPE_ATOMIC) { - - pipeline_tx_pkt(ev[i].mbuf); - ev[i].op = RTE_EVENT_OP_RELEASE; - w->processed_pkts++; - } else - pipeline_fwd_event(&ev[i], - RTE_SCHED_TYPE_ATOMIC); + rte_event_eth_tx_adapter_txq_set(ev[i].mbuf, 0); } - pipeline_event_enqueue_burst(dev, port, ev, nb_rx); + pipeline_event_tx_burst(dev, port, ev, nb_rx); + w->processed_pkts += nb_rx; } return 0; } -static int +static __rte_noinline int pipeline_atq_worker_single_stage_burst_fwd(void *arg) { - PIPELINE_WROKER_SINGLE_STAGE_BURST_INIT; - const uint8_t tx_queue = t->tx_service.queue_id; + PIPELINE_WORKER_SINGLE_STAGE_BURST_INIT; + const uint8_t *tx_queue = t->tx_evqueue_id; while (t->done == false) { uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev, @@ -112,23 +101,22 @@ pipeline_atq_worker_single_stage_burst_fwd(void *arg) for (i = 0; i < nb_rx; i++) { rte_prefetch0(ev[i + 1].mbuf); - ev[i].queue_id = tx_queue; + rte_event_eth_tx_adapter_txq_set(ev[i].mbuf, 0); + ev[i].queue_id = tx_queue[ev[i].mbuf->port]; pipeline_fwd_event(&ev[i], RTE_SCHED_TYPE_ATOMIC); - w->processed_pkts++; } pipeline_event_enqueue_burst(dev, port, ev, nb_rx); + w->processed_pkts += nb_rx; } return 0; } -static int +static __rte_noinline int pipeline_atq_worker_multi_stage_tx(void *arg) { - PIPELINE_WROKER_MULTI_STAGE_INIT; - const uint8_t nb_stages = t->opt->nb_stages; - + PIPELINE_WORKER_MULTI_STAGE_INIT; while (t->done == false) { uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0); @@ -141,29 +129,24 @@ pipeline_atq_worker_multi_stage_tx(void *arg) cq_id = ev.sub_event_type % nb_stages; if (cq_id == last_queue) { - if (ev.sched_type == RTE_SCHED_TYPE_ATOMIC) { - - pipeline_tx_pkt(ev.mbuf); - w->processed_pkts++; - continue; - } - pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC); - } else { - ev.sub_event_type++; - pipeline_fwd_event(&ev, sched_type_list[cq_id]); + pipeline_event_tx(dev, port, &ev); + w->processed_pkts++; + continue; } + ev.sub_event_type++; + pipeline_fwd_event(&ev, sched_type_list[cq_id]); pipeline_event_enqueue(dev, port, &ev); } + return 0; } -static int +static __rte_noinline int pipeline_atq_worker_multi_stage_fwd(void *arg) { - PIPELINE_WROKER_MULTI_STAGE_INIT; - const uint8_t nb_stages = t->opt->nb_stages; - const uint8_t tx_queue = t->tx_service.queue_id; + PIPELINE_WORKER_MULTI_STAGE_INIT; + const uint8_t *tx_queue = t->tx_evqueue_id; while (t->done == false) { uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0); @@ -176,9 +159,9 @@ pipeline_atq_worker_multi_stage_fwd(void *arg) cq_id = ev.sub_event_type % nb_stages; if (cq_id == last_queue) { - w->processed_pkts++; - ev.queue_id = tx_queue; + ev.queue_id = tx_queue[ev.mbuf->port]; pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC); + w->processed_pkts++; } else { ev.sub_event_type++; pipeline_fwd_event(&ev, sched_type_list[cq_id]); @@ -186,14 +169,14 @@ pipeline_atq_worker_multi_stage_fwd(void *arg) pipeline_event_enqueue(dev, port, &ev); } + return 0; } -static int +static __rte_noinline int pipeline_atq_worker_multi_stage_burst_tx(void *arg) { - PIPELINE_WROKER_MULTI_STAGE_BURST_INIT; - const uint8_t nb_stages = t->opt->nb_stages; + PIPELINE_WORKER_MULTI_STAGE_BURST_INIT; while (t->done == false) { uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev, @@ -209,34 +192,27 @@ pipeline_atq_worker_multi_stage_burst_tx(void *arg) cq_id = ev[i].sub_event_type % nb_stages; if (cq_id == last_queue) { - if (ev[i].sched_type == RTE_SCHED_TYPE_ATOMIC) { - - pipeline_tx_pkt(ev[i].mbuf); - ev[i].op = RTE_EVENT_OP_RELEASE; - w->processed_pkts++; - continue; - } - - pipeline_fwd_event(&ev[i], - RTE_SCHED_TYPE_ATOMIC); - } else { - ev[i].sub_event_type++; - pipeline_fwd_event(&ev[i], - sched_type_list[cq_id]); + pipeline_event_tx(dev, port, &ev[i]); + ev[i].op = RTE_EVENT_OP_RELEASE; + w->processed_pkts++; + continue; } + + ev[i].sub_event_type++; + pipeline_fwd_event(&ev[i], sched_type_list[cq_id]); } pipeline_event_enqueue_burst(dev, port, ev, nb_rx); } + return 0; } -static int +static __rte_noinline int pipeline_atq_worker_multi_stage_burst_fwd(void *arg) { - PIPELINE_WROKER_MULTI_STAGE_BURST_INIT; - const uint8_t nb_stages = t->opt->nb_stages; - const uint8_t tx_queue = t->tx_service.queue_id; + PIPELINE_WORKER_MULTI_STAGE_BURST_INIT; + const uint8_t *tx_queue = t->tx_evqueue_id; while (t->done == false) { uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev, @@ -253,7 +229,7 @@ pipeline_atq_worker_multi_stage_burst_fwd(void *arg) if (cq_id == last_queue) { w->processed_pkts++; - ev[i].queue_id = tx_queue; + ev[i].queue_id = tx_queue[ev[i].mbuf->port]; pipeline_fwd_event(&ev[i], RTE_SCHED_TYPE_ATOMIC); } else { @@ -265,6 +241,7 @@ pipeline_atq_worker_multi_stage_burst_fwd(void *arg) pipeline_event_enqueue_burst(dev, port, ev, nb_rx); } + return 0; } @@ -274,39 +251,36 @@ worker_wrapper(void *arg) struct worker_data *w = arg; struct evt_options *opt = w->t->opt; const bool burst = evt_has_burst_mode(w->dev_id); - const bool mt_safe = !w->t->mt_unsafe; + const bool internal_port = w->t->internal_port; const uint8_t nb_stages = opt->nb_stages; RTE_SET_USED(opt); if (nb_stages == 1) { - if (!burst && mt_safe) + if (!burst && internal_port) return pipeline_atq_worker_single_stage_tx(arg); - else if (!burst && !mt_safe) + else if (!burst && !internal_port) return pipeline_atq_worker_single_stage_fwd(arg); - else if (burst && mt_safe) + else if (burst && internal_port) return pipeline_atq_worker_single_stage_burst_tx(arg); - else if (burst && !mt_safe) + else if (burst && !internal_port) return pipeline_atq_worker_single_stage_burst_fwd(arg); } else { - if (!burst && mt_safe) + if (!burst && internal_port) return pipeline_atq_worker_multi_stage_tx(arg); - else if (!burst && !mt_safe) + else if (!burst && !internal_port) return pipeline_atq_worker_multi_stage_fwd(arg); - if (burst && mt_safe) + if (burst && internal_port) return pipeline_atq_worker_multi_stage_burst_tx(arg); - else if (burst && !mt_safe) + else if (burst && !internal_port) return pipeline_atq_worker_multi_stage_burst_fwd(arg); } + rte_panic("invalid worker\n"); } static int pipeline_atq_launch_lcores(struct evt_test *test, struct evt_options *opt) { - struct test_pipeline *t = evt_test_priv(test); - - if (t->mt_unsafe) - rte_service_component_runstate_set(t->tx_service.service_id, 1); return pipeline_launch_lcores(test, opt, worker_wrapper); } @@ -317,34 +291,38 @@ pipeline_atq_eventdev_setup(struct evt_test *test, struct evt_options *opt) int nb_ports; int nb_queues; uint8_t queue; - struct rte_event_dev_info info; - struct test_pipeline *t = evt_test_priv(test); - uint8_t tx_evqueue_id = 0; + uint8_t tx_evqueue_id[RTE_MAX_ETHPORTS]; uint8_t queue_arr[RTE_EVENT_MAX_QUEUES_PER_DEV]; uint8_t nb_worker_queues = 0; + uint8_t tx_evport_id = 0; + uint16_t prod = 0; + struct rte_event_dev_info info; + struct test_pipeline *t = evt_test_priv(test); nb_ports = evt_nr_active_lcores(opt->wlcores); nb_queues = rte_eth_dev_count_avail(); - /* One extra port and queueu for Tx service */ - if (t->mt_unsafe) { - tx_evqueue_id = nb_queues; - nb_ports++; - nb_queues++; + memset(tx_evqueue_id, 0, sizeof(uint8_t) * RTE_MAX_ETHPORTS); + memset(queue_arr, 0, sizeof(uint8_t) * RTE_EVENT_MAX_QUEUES_PER_DEV); + /* One queue for Tx adapter per port */ + if (!t->internal_port) { + RTE_ETH_FOREACH_DEV(prod) { + tx_evqueue_id[prod] = nb_queues; + nb_queues++; + } } - rte_event_dev_info_get(opt->dev_id, &info); const struct rte_event_dev_config config = { - .nb_event_queues = nb_queues, - .nb_event_ports = nb_ports, - .nb_events_limit = info.max_num_events, - .nb_event_queue_flows = opt->nb_flows, - .nb_event_port_dequeue_depth = - info.max_event_port_dequeue_depth, - .nb_event_port_enqueue_depth = - info.max_event_port_enqueue_depth, + .nb_event_queues = nb_queues, + .nb_event_ports = nb_ports, + .nb_events_limit = info.max_num_events, + .nb_event_queue_flows = opt->nb_flows, + .nb_event_port_dequeue_depth = + info.max_event_port_dequeue_depth, + .nb_event_port_enqueue_depth = + info.max_event_port_enqueue_depth, }; ret = rte_event_dev_configure(opt->dev_id, &config); if (ret) { @@ -353,21 +331,23 @@ pipeline_atq_eventdev_setup(struct evt_test *test, struct evt_options *opt) } struct rte_event_queue_conf q_conf = { - .priority = RTE_EVENT_DEV_PRIORITY_NORMAL, - .nb_atomic_flows = opt->nb_flows, - .nb_atomic_order_sequences = opt->nb_flows, + .priority = RTE_EVENT_DEV_PRIORITY_NORMAL, + .nb_atomic_flows = opt->nb_flows, + .nb_atomic_order_sequences = opt->nb_flows, }; /* queue configurations */ for (queue = 0; queue < nb_queues; queue++) { q_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_ALL_TYPES; - if (t->mt_unsafe) { - if (queue == tx_evqueue_id) { - q_conf.event_queue_cfg = - RTE_EVENT_QUEUE_CFG_SINGLE_LINK; - } else { - queue_arr[nb_worker_queues] = queue; - nb_worker_queues++; + if (!t->internal_port) { + RTE_ETH_FOREACH_DEV(prod) { + if (queue == tx_evqueue_id[prod]) { + q_conf.event_queue_cfg = + RTE_EVENT_QUEUE_CFG_SINGLE_LINK; + } else { + queue_arr[nb_worker_queues] = queue; + nb_worker_queues++; + } } } @@ -383,20 +363,15 @@ pipeline_atq_eventdev_setup(struct evt_test *test, struct evt_options *opt) /* port configuration */ const struct rte_event_port_conf p_conf = { - .dequeue_depth = opt->wkr_deq_dep, - .enqueue_depth = info.max_event_port_dequeue_depth, - .new_event_threshold = info.max_num_events, + .dequeue_depth = opt->wkr_deq_dep, + .enqueue_depth = info.max_event_port_dequeue_depth, + .new_event_threshold = info.max_num_events, }; - if (t->mt_unsafe) { + if (!t->internal_port) ret = pipeline_event_port_setup(test, opt, queue_arr, nb_worker_queues, p_conf); - if (ret) - return ret; - - ret = pipeline_event_tx_service_setup(test, opt, tx_evqueue_id, - nb_ports - 1, p_conf); - } else + else ret = pipeline_event_port_setup(test, opt, NULL, nb_queues, p_conf); @@ -408,30 +383,32 @@ pipeline_atq_eventdev_setup(struct evt_test *test, struct evt_options *opt) * * eth_dev_count = 2, nb_stages = 2, atq mode * - * Multi thread safe : + * eth0, eth1 have Internal port capability : * queues = 2 * stride = 1 * * event queue pipelines: - * eth0 -> q0 ->tx - * eth1 -> q1 ->tx + * eth0 -> q0 ->Tx + * eth1 -> q1 ->Tx * * q0, q1 are configured as ATQ so, all the different stages can * be enqueued on the same queue. * - * Multi thread unsafe : - * queues = 3 + * eth0, eth1 use Tx adapters service core : + * queues = 4 * stride = 1 * * event queue pipelines: - * eth0 -> q0 - * } (q3->tx) Tx service - * eth1 -> q1 + * eth0 -> q0 -> q2 -> Tx + * eth1 -> q1 -> q3 -> Tx * - * q0,q1 are configured as stated above. - * q3 configured as SINGLE_LINK|ATOMIC. + * q0, q1 are configured as stated above. + * q2, q3 configured as SINGLE_LINK. */ ret = pipeline_event_rx_adapter_setup(opt, 1, p_conf); + if (ret) + return ret; + ret = pipeline_event_tx_adapter_setup(opt, p_conf); if (ret) return ret; @@ -445,12 +422,58 @@ pipeline_atq_eventdev_setup(struct evt_test *test, struct evt_options *opt) } } + /* Connect the tx_evqueue_id to the Tx adapter port */ + if (!t->internal_port) { + RTE_ETH_FOREACH_DEV(prod) { + ret = rte_event_eth_tx_adapter_event_port_get(prod, + &tx_evport_id); + if (ret) { + evt_err("Unable to get Tx adapter[%d]", prod); + return ret; + } + + if (rte_event_port_link(opt->dev_id, tx_evport_id, + &tx_evqueue_id[prod], + NULL, 1) != 1) { + evt_err("Unable to link Tx adptr[%d] evprt[%d]", + prod, tx_evport_id); + return ret; + } + } + } + + RTE_ETH_FOREACH_DEV(prod) { + ret = rte_eth_dev_start(prod); + if (ret) { + evt_err("Ethernet dev [%d] failed to start." + " Using synthetic producer", prod); + return ret; + } + } + ret = rte_event_dev_start(opt->dev_id); if (ret) { evt_err("failed to start eventdev %d", opt->dev_id); return ret; } + RTE_ETH_FOREACH_DEV(prod) { + ret = rte_event_eth_rx_adapter_start(prod); + if (ret) { + evt_err("Rx adapter[%d] start failed", prod); + return ret; + } + + ret = rte_event_eth_tx_adapter_start(prod); + if (ret) { + evt_err("Tx adapter[%d] start failed", prod); + return ret; + } + } + + memcpy(t->tx_evqueue_id, tx_evqueue_id, sizeof(uint8_t) * + RTE_MAX_ETHPORTS); + return 0; } diff --git a/app/test-eventdev/test_pipeline_common.c b/app/test-eventdev/test_pipeline_common.c index a54068df..d07fa882 100644 --- a/app/test-eventdev/test_pipeline_common.c +++ b/app/test-eventdev/test_pipeline_common.c @@ -5,58 +5,6 @@ #include "test_pipeline_common.h" -static int32_t -pipeline_event_tx_burst_service_func(void *args) -{ - - int i; - struct tx_service_data *tx = args; - const uint8_t dev = tx->dev_id; - const uint8_t port = tx->port_id; - struct rte_event ev[BURST_SIZE + 1]; - - uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev, BURST_SIZE, 0); - - if (!nb_rx) { - for (i = 0; i < tx->nb_ethports; i++) - rte_eth_tx_buffer_flush(i, 0, tx->tx_buf[i]); - return 0; - } - - for (i = 0; i < nb_rx; i++) { - struct rte_mbuf *m = ev[i].mbuf; - rte_eth_tx_buffer(m->port, 0, tx->tx_buf[m->port], m); - } - tx->processed_pkts += nb_rx; - - return 0; -} - -static int32_t -pipeline_event_tx_service_func(void *args) -{ - - int i; - struct tx_service_data *tx = args; - const uint8_t dev = tx->dev_id; - const uint8_t port = tx->port_id; - struct rte_event ev; - - uint16_t nb_rx = rte_event_dequeue_burst(dev, port, &ev, 1, 0); - - if (!nb_rx) { - for (i = 0; i < tx->nb_ethports; i++) - rte_eth_tx_buffer_flush(i, 0, tx->tx_buf[i]); - return 0; - } - - struct rte_mbuf *m = ev.mbuf; - rte_eth_tx_buffer(m->port, 0, tx->tx_buf[m->port], m); - tx->processed_pkts++; - - return 0; -} - int pipeline_test_result(struct evt_test *test, struct evt_options *opt) { @@ -65,12 +13,12 @@ pipeline_test_result(struct evt_test *test, struct evt_options *opt) uint64_t total = 0; struct test_pipeline *t = evt_test_priv(test); - printf("Packet distribution across worker cores :\n"); + evt_info("Packet distribution across worker cores :"); for (i = 0; i < t->nb_workers; i++) total += t->worker[i].processed_pkts; for (i = 0; i < t->nb_workers; i++) - printf("Worker %d packets: "CLGRN"%"PRIx64" "CLNRM"percentage:" - CLGRN" %3.2f\n"CLNRM, i, + evt_info("Worker %d packets: "CLGRN"%"PRIx64""CLNRM" percentage:" + CLGRN" %3.2f"CLNRM, i, t->worker[i].processed_pkts, (((double)t->worker[i].processed_pkts)/total) * 100); @@ -97,11 +45,8 @@ processed_pkts(struct test_pipeline *t) uint64_t total = 0; rte_smp_rmb(); - if (t->mt_unsafe) - total = t->tx_service.processed_pkts; - else - for (i = 0; i < t->nb_workers; i++) - total += t->worker[i].processed_pkts; + for (i = 0; i < t->nb_workers; i++) + total += t->worker[i].processed_pkts; return total; } @@ -215,14 +160,12 @@ pipeline_ethdev_setup(struct evt_test *test, struct evt_options *opt) { uint16_t i; uint8_t nb_queues = 1; - uint8_t mt_state = 0; struct test_pipeline *t = evt_test_priv(test); struct rte_eth_rxconf rx_conf; struct rte_eth_conf port_conf = { .rxmode = { .mq_mode = ETH_MQ_RX_RSS, .max_rx_pkt_len = ETHER_MAX_LEN, - .offloads = DEV_RX_OFFLOAD_CRC_STRIP, }, .rx_adv_conf = { .rss_conf = { @@ -234,17 +177,21 @@ pipeline_ethdev_setup(struct evt_test *test, struct evt_options *opt) RTE_SET_USED(opt); if (!rte_eth_dev_count_avail()) { - evt_err("No ethernet ports found.\n"); + evt_err("No ethernet ports found."); return -ENODEV; } + t->internal_port = 1; RTE_ETH_FOREACH_DEV(i) { struct rte_eth_dev_info dev_info; struct rte_eth_conf local_port_conf = port_conf; + uint32_t caps = 0; + + rte_event_eth_tx_adapter_caps_get(opt->dev_id, i, &caps); + if (!(caps & RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT)) + t->internal_port = 0; rte_eth_dev_info_get(i, &dev_info); - mt_state = !(dev_info.tx_offload_capa & - DEV_TX_OFFLOAD_MT_LOCKFREE); rx_conf = dev_info.default_rxconf; rx_conf.offloads = port_conf.rxmode.offloads; @@ -253,7 +200,7 @@ pipeline_ethdev_setup(struct evt_test *test, struct evt_options *opt) if (local_port_conf.rx_adv_conf.rss_conf.rss_hf != port_conf.rx_adv_conf.rss_conf.rss_hf) { evt_info("Port %u modified RSS hash function based on hardware support," - "requested:%#"PRIx64" configured:%#"PRIx64"\n", + "requested:%#"PRIx64" configured:%#"PRIx64"", i, port_conf.rx_adv_conf.rss_conf.rss_hf, local_port_conf.rx_adv_conf.rss_conf.rss_hf); @@ -262,28 +209,23 @@ pipeline_ethdev_setup(struct evt_test *test, struct evt_options *opt) if (rte_eth_dev_configure(i, nb_queues, nb_queues, &local_port_conf) < 0) { - evt_err("Failed to configure eth port [%d]\n", i); + evt_err("Failed to configure eth port [%d]", i); return -EINVAL; } if (rte_eth_rx_queue_setup(i, 0, NB_RX_DESC, rte_socket_id(), &rx_conf, t->pool) < 0) { - evt_err("Failed to setup eth port [%d] rx_queue: %d.\n", + evt_err("Failed to setup eth port [%d] rx_queue: %d.", i, 0); return -EINVAL; } if (rte_eth_tx_queue_setup(i, 0, NB_TX_DESC, rte_socket_id(), NULL) < 0) { - evt_err("Failed to setup eth port [%d] tx_queue: %d.\n", + evt_err("Failed to setup eth port [%d] tx_queue: %d.", i, 0); return -EINVAL; } - t->mt_unsafe |= mt_state; - t->tx_service.tx_buf[i] = - rte_malloc(NULL, RTE_ETH_TX_BUFFER_SIZE(BURST_SIZE), 0); - if (t->tx_service.tx_buf[i] == NULL) - rte_panic("Unable to allocate Tx buffer memory."); rte_eth_promiscuous_enable(i); } @@ -295,7 +237,6 @@ pipeline_event_port_setup(struct evt_test *test, struct evt_options *opt, uint8_t *queue_arr, uint8_t nb_queues, const struct rte_event_port_conf p_conf) { - int i; int ret; uint8_t port; struct test_pipeline *t = evt_test_priv(test); @@ -316,23 +257,15 @@ pipeline_event_port_setup(struct evt_test *test, struct evt_options *opt, return ret; } - if (queue_arr == NULL) { - if (rte_event_port_link(opt->dev_id, port, NULL, NULL, - 0) != nb_queues) - goto link_fail; - } else { - for (i = 0; i < nb_queues; i++) { - if (rte_event_port_link(opt->dev_id, port, - &queue_arr[i], NULL, 1) != 1) - goto link_fail; - } - } + if (rte_event_port_link(opt->dev_id, port, queue_arr, NULL, + nb_queues) != nb_queues) + goto link_fail; } return 0; link_fail: - evt_err("failed to link all queues to port %d", port); + evt_err("failed to link queues to port %d", port); return -EINVAL; } @@ -380,85 +313,69 @@ pipeline_event_rx_adapter_setup(struct evt_options *opt, uint8_t stride, ret = evt_service_setup(service_id); if (ret) { evt_err("Failed to setup service core" - " for Rx adapter\n"); + " for Rx adapter"); return ret; } } - ret = rte_eth_dev_start(prod); - if (ret) { - evt_err("Ethernet dev [%d] failed to start." - " Using synthetic producer", prod); - return ret; - } - - ret = rte_event_eth_rx_adapter_start(prod); - if (ret) { - evt_err("Rx adapter[%d] start failed", prod); - return ret; - } - printf("%s: Port[%d] using Rx adapter[%d] started\n", __func__, - prod, prod); + evt_info("Port[%d] using Rx adapter[%d] configured", prod, + prod); } return ret; } int -pipeline_event_tx_service_setup(struct evt_test *test, struct evt_options *opt, - uint8_t tx_queue_id, uint8_t tx_port_id, - const struct rte_event_port_conf p_conf) +pipeline_event_tx_adapter_setup(struct evt_options *opt, + struct rte_event_port_conf port_conf) { int ret; - struct rte_service_spec serv; - struct test_pipeline *t = evt_test_priv(test); - struct tx_service_data *tx = &t->tx_service; + uint16_t consm; - ret = rte_event_port_setup(opt->dev_id, tx_port_id, &p_conf); - if (ret) { - evt_err("failed to setup port %d", tx_port_id); - return ret; - } + RTE_ETH_FOREACH_DEV(consm) { + uint32_t cap; - if (rte_event_port_link(opt->dev_id, tx_port_id, &tx_queue_id, - NULL, 1) != 1) { - evt_err("failed to link queues to port %d", tx_port_id); - return -EINVAL; - } + ret = rte_event_eth_tx_adapter_caps_get(opt->dev_id, + consm, &cap); + if (ret) { + evt_err("failed to get event tx adapter[%d] caps", + consm); + return ret; + } - tx->dev_id = opt->dev_id; - tx->queue_id = tx_queue_id; - tx->port_id = tx_port_id; - tx->nb_ethports = rte_eth_dev_count_avail(); - tx->t = t; - - /* Register Tx service */ - memset(&serv, 0, sizeof(struct rte_service_spec)); - snprintf(serv.name, sizeof(serv.name), "Tx_service"); - - if (evt_has_burst_mode(opt->dev_id)) - serv.callback = pipeline_event_tx_burst_service_func; - else - serv.callback = pipeline_event_tx_service_func; - - serv.callback_userdata = (void *)tx; - ret = rte_service_component_register(&serv, &tx->service_id); - if (ret) { - evt_err("failed to register Tx service"); - return ret; - } + ret = rte_event_eth_tx_adapter_create(consm, opt->dev_id, + &port_conf); + if (ret) { + evt_err("failed to create tx adapter[%d]", consm); + return ret; + } - ret = evt_service_setup(tx->service_id); - if (ret) { - evt_err("Failed to setup service core for Tx service\n"); - return ret; - } + ret = rte_event_eth_tx_adapter_queue_add(consm, consm, -1); + if (ret) { + evt_err("failed to add tx queues to adapter[%d]", + consm); + return ret; + } - rte_service_runstate_set(tx->service_id, 1); + if (!(cap & RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT)) { + uint32_t service_id; - return 0; -} + rte_event_eth_tx_adapter_service_id_get(consm, + &service_id); + ret = evt_service_setup(service_id); + if (ret) { + evt_err("Failed to setup service core" + " for Tx adapter\n"); + return ret; + } + } + + evt_info("Port[%d] using Tx adapter[%d] Configured", consm, + consm); + } + return ret; +} void pipeline_ethdev_destroy(struct evt_test *test, struct evt_options *opt) @@ -466,16 +383,10 @@ pipeline_ethdev_destroy(struct evt_test *test, struct evt_options *opt) uint16_t i; RTE_SET_USED(test); RTE_SET_USED(opt); - struct test_pipeline *t = evt_test_priv(test); - - if (t->mt_unsafe) { - rte_service_component_runstate_set(t->tx_service.service_id, 0); - rte_service_runstate_set(t->tx_service.service_id, 0); - rte_service_component_unregister(t->tx_service.service_id); - } RTE_ETH_FOREACH_DEV(i) { rte_event_eth_rx_adapter_stop(i); + rte_event_eth_tx_adapter_stop(i); rte_eth_dev_stop(i); } } @@ -485,7 +396,6 @@ pipeline_eventdev_destroy(struct evt_test *test, struct evt_options *opt) { RTE_SET_USED(test); - rte_event_dev_stop(opt->dev_id); rte_event_dev_close(opt->dev_id); } diff --git a/app/test-eventdev/test_pipeline_common.h b/app/test-eventdev/test_pipeline_common.h index 5fb91607..0440b9e2 100644 --- a/app/test-eventdev/test_pipeline_common.h +++ b/app/test-eventdev/test_pipeline_common.h @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -35,52 +36,41 @@ struct worker_data { struct test_pipeline *t; } __rte_cache_aligned; -struct tx_service_data { - uint8_t dev_id; - uint8_t queue_id; - uint8_t port_id; - uint32_t service_id; - uint64_t processed_pkts; - uint16_t nb_ethports; - struct rte_eth_dev_tx_buffer *tx_buf[RTE_MAX_ETHPORTS]; - struct test_pipeline *t; -} __rte_cache_aligned; - struct test_pipeline { /* Don't change the offset of "done". Signal handler use this memory * to terminate all lcores work. */ int done; uint8_t nb_workers; - uint8_t mt_unsafe; + uint8_t internal_port; + uint8_t tx_evqueue_id[RTE_MAX_ETHPORTS]; enum evt_test_result result; uint32_t nb_flows; uint64_t outstand_pkts; struct rte_mempool *pool; struct worker_data worker[EVT_MAX_PORTS]; - struct tx_service_data tx_service; struct evt_options *opt; uint8_t sched_type_list[EVT_MAX_STAGES] __rte_cache_aligned; } __rte_cache_aligned; #define BURST_SIZE 16 -#define PIPELINE_WROKER_SINGLE_STAGE_INIT \ +#define PIPELINE_WORKER_SINGLE_STAGE_INIT \ struct worker_data *w = arg; \ struct test_pipeline *t = w->t; \ const uint8_t dev = w->dev_id; \ const uint8_t port = w->port_id; \ - struct rte_event ev + struct rte_event ev __rte_cache_aligned -#define PIPELINE_WROKER_SINGLE_STAGE_BURST_INIT \ +#define PIPELINE_WORKER_SINGLE_STAGE_BURST_INIT \ int i; \ struct worker_data *w = arg; \ struct test_pipeline *t = w->t; \ const uint8_t dev = w->dev_id; \ const uint8_t port = w->port_id; \ - struct rte_event ev[BURST_SIZE + 1] + struct rte_event ev[BURST_SIZE + 1] __rte_cache_aligned -#define PIPELINE_WROKER_MULTI_STAGE_INIT \ +#define PIPELINE_WORKER_MULTI_STAGE_INIT \ struct worker_data *w = arg; \ struct test_pipeline *t = w->t; \ uint8_t cq_id; \ @@ -88,10 +78,11 @@ struct test_pipeline { const uint8_t port = w->port_id; \ const uint8_t last_queue = t->opt->nb_stages - 1; \ uint8_t *const sched_type_list = &t->sched_type_list[0]; \ - struct rte_event ev + const uint8_t nb_stages = t->opt->nb_stages + 1; \ + struct rte_event ev __rte_cache_aligned -#define PIPELINE_WROKER_MULTI_STAGE_BURST_INIT \ - int i; \ +#define PIPELINE_WORKER_MULTI_STAGE_BURST_INIT \ + int i; \ struct worker_data *w = arg; \ struct test_pipeline *t = w->t; \ uint8_t cq_id; \ @@ -99,7 +90,8 @@ struct test_pipeline { const uint8_t port = w->port_id; \ const uint8_t last_queue = t->opt->nb_stages - 1; \ uint8_t *const sched_type_list = &t->sched_type_list[0]; \ - struct rte_event ev[BURST_SIZE + 1] + const uint8_t nb_stages = t->opt->nb_stages + 1; \ + struct rte_event ev[BURST_SIZE + 1] __rte_cache_aligned static __rte_always_inline void pipeline_fwd_event(struct rte_event *ev, uint8_t sched) @@ -109,6 +101,28 @@ pipeline_fwd_event(struct rte_event *ev, uint8_t sched) ev->sched_type = sched; } +static __rte_always_inline void +pipeline_event_tx(const uint8_t dev, const uint8_t port, + struct rte_event * const ev) +{ + rte_event_eth_tx_adapter_txq_set(ev->mbuf, 0); + while (!rte_event_eth_tx_adapter_enqueue(dev, port, ev, 1)) + rte_pause(); +} + +static __rte_always_inline void +pipeline_event_tx_burst(const uint8_t dev, const uint8_t port, + struct rte_event *ev, const uint16_t nb_rx) +{ + uint16_t enq; + + enq = rte_event_eth_tx_adapter_enqueue(dev, port, ev, nb_rx); + while (enq < nb_rx) { + enq += rte_event_eth_tx_adapter_enqueue(dev, port, + ev + enq, nb_rx - enq); + } +} + static __rte_always_inline void pipeline_event_enqueue(const uint8_t dev, const uint8_t port, struct rte_event *ev) @@ -130,13 +144,6 @@ pipeline_event_enqueue_burst(const uint8_t dev, const uint8_t port, } } -static __rte_always_inline void -pipeline_tx_pkt(struct rte_mbuf *mbuf) -{ - while (rte_eth_tx_burst(mbuf->port, 0, &mbuf, 1) != 1) - rte_pause(); -} - static inline int pipeline_nb_event_ports(struct evt_options *opt) { @@ -149,9 +156,8 @@ int pipeline_test_setup(struct evt_test *test, struct evt_options *opt); int pipeline_ethdev_setup(struct evt_test *test, struct evt_options *opt); int pipeline_event_rx_adapter_setup(struct evt_options *opt, uint8_t stride, struct rte_event_port_conf prod_conf); -int pipeline_event_tx_service_setup(struct evt_test *test, - struct evt_options *opt, uint8_t tx_queue_id, - uint8_t tx_port_id, const struct rte_event_port_conf p_conf); +int pipeline_event_tx_adapter_setup(struct evt_options *opt, + struct rte_event_port_conf prod_conf); int pipeline_mempool_setup(struct evt_test *test, struct evt_options *opt); int pipeline_event_port_setup(struct evt_test *test, struct evt_options *opt, uint8_t *queue_arr, uint8_t nb_queues, diff --git a/app/test-eventdev/test_pipeline_queue.c b/app/test-eventdev/test_pipeline_queue.c index ca5f4578..25217008 100644 --- a/app/test-eventdev/test_pipeline_queue.c +++ b/app/test-eventdev/test_pipeline_queue.c @@ -15,10 +15,10 @@ pipeline_queue_nb_event_queues(struct evt_options *opt) return (eth_count * opt->nb_stages) + eth_count; } -static int +static __rte_noinline int pipeline_queue_worker_single_stage_tx(void *arg) { - PIPELINE_WROKER_SINGLE_STAGE_INIT; + PIPELINE_WORKER_SINGLE_STAGE_INIT; while (t->done == false) { uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0); @@ -29,7 +29,7 @@ pipeline_queue_worker_single_stage_tx(void *arg) } if (ev.sched_type == RTE_SCHED_TYPE_ATOMIC) { - pipeline_tx_pkt(ev.mbuf); + pipeline_event_tx(dev, port, &ev); w->processed_pkts++; } else { ev.queue_id++; @@ -41,11 +41,11 @@ pipeline_queue_worker_single_stage_tx(void *arg) return 0; } -static int +static __rte_noinline int pipeline_queue_worker_single_stage_fwd(void *arg) { - PIPELINE_WROKER_SINGLE_STAGE_INIT; - const uint8_t tx_queue = t->tx_service.queue_id; + PIPELINE_WORKER_SINGLE_STAGE_INIT; + const uint8_t *tx_queue = t->tx_evqueue_id; while (t->done == false) { uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0); @@ -55,7 +55,8 @@ pipeline_queue_worker_single_stage_fwd(void *arg) continue; } - ev.queue_id = tx_queue; + ev.queue_id = tx_queue[ev.mbuf->port]; + rte_event_eth_tx_adapter_txq_set(ev.mbuf, 0); pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC); pipeline_event_enqueue(dev, port, &ev); w->processed_pkts++; @@ -64,10 +65,10 @@ pipeline_queue_worker_single_stage_fwd(void *arg) return 0; } -static int +static __rte_noinline int pipeline_queue_worker_single_stage_burst_tx(void *arg) { - PIPELINE_WROKER_SINGLE_STAGE_BURST_INIT; + PIPELINE_WORKER_SINGLE_STAGE_BURST_INIT; while (t->done == false) { uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev, @@ -81,8 +82,7 @@ pipeline_queue_worker_single_stage_burst_tx(void *arg) for (i = 0; i < nb_rx; i++) { rte_prefetch0(ev[i + 1].mbuf); if (ev[i].sched_type == RTE_SCHED_TYPE_ATOMIC) { - - pipeline_tx_pkt(ev[i].mbuf); + pipeline_event_tx(dev, port, &ev[i]); ev[i].op = RTE_EVENT_OP_RELEASE; w->processed_pkts++; } else { @@ -98,11 +98,11 @@ pipeline_queue_worker_single_stage_burst_tx(void *arg) return 0; } -static int +static __rte_noinline int pipeline_queue_worker_single_stage_burst_fwd(void *arg) { - PIPELINE_WROKER_SINGLE_STAGE_BURST_INIT; - const uint8_t tx_queue = t->tx_service.queue_id; + PIPELINE_WORKER_SINGLE_STAGE_BURST_INIT; + const uint8_t *tx_queue = t->tx_evqueue_id; while (t->done == false) { uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev, @@ -115,23 +115,24 @@ pipeline_queue_worker_single_stage_burst_fwd(void *arg) for (i = 0; i < nb_rx; i++) { rte_prefetch0(ev[i + 1].mbuf); - ev[i].queue_id = tx_queue; + ev[i].queue_id = tx_queue[ev[i].mbuf->port]; + rte_event_eth_tx_adapter_txq_set(ev[i].mbuf, 0); pipeline_fwd_event(&ev[i], RTE_SCHED_TYPE_ATOMIC); - w->processed_pkts++; } pipeline_event_enqueue_burst(dev, port, ev, nb_rx); + w->processed_pkts += nb_rx; } return 0; } -static int +static __rte_noinline int pipeline_queue_worker_multi_stage_tx(void *arg) { - PIPELINE_WROKER_MULTI_STAGE_INIT; - const uint8_t nb_stages = t->opt->nb_stages + 1; + PIPELINE_WORKER_MULTI_STAGE_INIT; + const uint8_t *tx_queue = t->tx_evqueue_id; while (t->done == false) { uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0); @@ -143,31 +144,27 @@ pipeline_queue_worker_multi_stage_tx(void *arg) cq_id = ev.queue_id % nb_stages; - if (cq_id >= last_queue) { - if (ev.sched_type == RTE_SCHED_TYPE_ATOMIC) { - - pipeline_tx_pkt(ev.mbuf); - w->processed_pkts++; - continue; - } - ev.queue_id += (cq_id == last_queue) ? 1 : 0; - pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC); - } else { - ev.queue_id++; - pipeline_fwd_event(&ev, sched_type_list[cq_id]); + if (ev.queue_id == tx_queue[ev.mbuf->port]) { + pipeline_event_tx(dev, port, &ev); + w->processed_pkts++; + continue; } + ev.queue_id++; + pipeline_fwd_event(&ev, cq_id != last_queue ? + sched_type_list[cq_id] : + RTE_SCHED_TYPE_ATOMIC); pipeline_event_enqueue(dev, port, &ev); } + return 0; } -static int +static __rte_noinline int pipeline_queue_worker_multi_stage_fwd(void *arg) { - PIPELINE_WROKER_MULTI_STAGE_INIT; - const uint8_t nb_stages = t->opt->nb_stages + 1; - const uint8_t tx_queue = t->tx_service.queue_id; + PIPELINE_WORKER_MULTI_STAGE_INIT; + const uint8_t *tx_queue = t->tx_evqueue_id; while (t->done == false) { uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0); @@ -180,7 +177,8 @@ pipeline_queue_worker_multi_stage_fwd(void *arg) cq_id = ev.queue_id % nb_stages; if (cq_id == last_queue) { - ev.queue_id = tx_queue; + ev.queue_id = tx_queue[ev.mbuf->port]; + rte_event_eth_tx_adapter_txq_set(ev.mbuf, 0); pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC); w->processed_pkts++; } else { @@ -190,14 +188,15 @@ pipeline_queue_worker_multi_stage_fwd(void *arg) pipeline_event_enqueue(dev, port, &ev); } + return 0; } -static int +static __rte_noinline int pipeline_queue_worker_multi_stage_burst_tx(void *arg) { - PIPELINE_WROKER_MULTI_STAGE_BURST_INIT; - const uint8_t nb_stages = t->opt->nb_stages + 1; + PIPELINE_WORKER_MULTI_STAGE_BURST_INIT; + const uint8_t *tx_queue = t->tx_evqueue_id; while (t->done == false) { uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev, @@ -212,37 +211,30 @@ pipeline_queue_worker_multi_stage_burst_tx(void *arg) rte_prefetch0(ev[i + 1].mbuf); cq_id = ev[i].queue_id % nb_stages; - if (cq_id >= last_queue) { - if (ev[i].sched_type == RTE_SCHED_TYPE_ATOMIC) { - - pipeline_tx_pkt(ev[i].mbuf); - ev[i].op = RTE_EVENT_OP_RELEASE; - w->processed_pkts++; - continue; - } - - ev[i].queue_id += (cq_id == last_queue) ? 1 : 0; - pipeline_fwd_event(&ev[i], - RTE_SCHED_TYPE_ATOMIC); - } else { - ev[i].queue_id++; - pipeline_fwd_event(&ev[i], - sched_type_list[cq_id]); + if (ev[i].queue_id == tx_queue[ev[i].mbuf->port]) { + pipeline_event_tx(dev, port, &ev[i]); + ev[i].op = RTE_EVENT_OP_RELEASE; + w->processed_pkts++; + continue; } + ev[i].queue_id++; + pipeline_fwd_event(&ev[i], cq_id != last_queue ? + sched_type_list[cq_id] : + RTE_SCHED_TYPE_ATOMIC); } pipeline_event_enqueue_burst(dev, port, ev, nb_rx); } + return 0; } -static int +static __rte_noinline int pipeline_queue_worker_multi_stage_burst_fwd(void *arg) { - PIPELINE_WROKER_MULTI_STAGE_BURST_INIT; - const uint8_t nb_stages = t->opt->nb_stages + 1; - const uint8_t tx_queue = t->tx_service.queue_id; + PIPELINE_WORKER_MULTI_STAGE_BURST_INIT; + const uint8_t *tx_queue = t->tx_evqueue_id; while (t->done == false) { uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev, @@ -258,7 +250,8 @@ pipeline_queue_worker_multi_stage_burst_fwd(void *arg) cq_id = ev[i].queue_id % nb_stages; if (cq_id == last_queue) { - ev[i].queue_id = tx_queue; + ev[i].queue_id = tx_queue[ev[i].mbuf->port]; + rte_event_eth_tx_adapter_txq_set(ev[i].mbuf, 0); pipeline_fwd_event(&ev[i], RTE_SCHED_TYPE_ATOMIC); w->processed_pkts++; @@ -271,6 +264,7 @@ pipeline_queue_worker_multi_stage_burst_fwd(void *arg) pipeline_event_enqueue_burst(dev, port, ev, nb_rx); } + return 0; } @@ -280,28 +274,28 @@ worker_wrapper(void *arg) struct worker_data *w = arg; struct evt_options *opt = w->t->opt; const bool burst = evt_has_burst_mode(w->dev_id); - const bool mt_safe = !w->t->mt_unsafe; + const bool internal_port = w->t->internal_port; const uint8_t nb_stages = opt->nb_stages; RTE_SET_USED(opt); if (nb_stages == 1) { - if (!burst && mt_safe) + if (!burst && internal_port) return pipeline_queue_worker_single_stage_tx(arg); - else if (!burst && !mt_safe) + else if (!burst && !internal_port) return pipeline_queue_worker_single_stage_fwd(arg); - else if (burst && mt_safe) + else if (burst && internal_port) return pipeline_queue_worker_single_stage_burst_tx(arg); - else if (burst && !mt_safe) + else if (burst && !internal_port) return pipeline_queue_worker_single_stage_burst_fwd( arg); } else { - if (!burst && mt_safe) + if (!burst && internal_port) return pipeline_queue_worker_multi_stage_tx(arg); - else if (!burst && !mt_safe) + else if (!burst && !internal_port) return pipeline_queue_worker_multi_stage_fwd(arg); - else if (burst && mt_safe) + else if (burst && internal_port) return pipeline_queue_worker_multi_stage_burst_tx(arg); - else if (burst && !mt_safe) + else if (burst && !internal_port) return pipeline_queue_worker_multi_stage_burst_fwd(arg); } @@ -311,10 +305,6 @@ worker_wrapper(void *arg) static int pipeline_queue_launch_lcores(struct evt_test *test, struct evt_options *opt) { - struct test_pipeline *t = evt_test_priv(test); - - if (t->mt_unsafe) - rte_service_component_runstate_set(t->tx_service.service_id, 1); return pipeline_launch_lcores(test, opt, worker_wrapper); } @@ -326,25 +316,24 @@ pipeline_queue_eventdev_setup(struct evt_test *test, struct evt_options *opt) int nb_queues; int nb_stages = opt->nb_stages; uint8_t queue; - struct rte_event_dev_info info; - struct test_pipeline *t = evt_test_priv(test); - uint8_t tx_evqueue_id = 0; + uint8_t tx_evport_id = 0; + uint8_t tx_evqueue_id[RTE_MAX_ETHPORTS]; uint8_t queue_arr[RTE_EVENT_MAX_QUEUES_PER_DEV]; uint8_t nb_worker_queues = 0; + uint16_t prod = 0; + struct rte_event_dev_info info; + struct test_pipeline *t = evt_test_priv(test); nb_ports = evt_nr_active_lcores(opt->wlcores); nb_queues = rte_eth_dev_count_avail() * (nb_stages); - /* Extra port for Tx service. */ - if (t->mt_unsafe) { - tx_evqueue_id = nb_queues; - nb_ports++; - nb_queues++; - } else - nb_queues += rte_eth_dev_count_avail(); + /* One queue for Tx adapter per port */ + nb_queues += rte_eth_dev_count_avail(); - rte_event_dev_info_get(opt->dev_id, &info); + memset(tx_evqueue_id, 0, sizeof(uint8_t) * RTE_MAX_ETHPORTS); + memset(queue_arr, 0, sizeof(uint8_t) * RTE_EVENT_MAX_QUEUES_PER_DEV); + rte_event_dev_info_get(opt->dev_id, &info); const struct rte_event_dev_config config = { .nb_event_queues = nb_queues, .nb_event_ports = nb_ports, @@ -370,24 +359,19 @@ pipeline_queue_eventdev_setup(struct evt_test *test, struct evt_options *opt) for (queue = 0; queue < nb_queues; queue++) { uint8_t slot; - if (!t->mt_unsafe) { - slot = queue % (nb_stages + 1); - q_conf.schedule_type = slot == nb_stages ? - RTE_SCHED_TYPE_ATOMIC : - opt->sched_type_list[slot]; - } else { - slot = queue % nb_stages; - - if (queue == tx_evqueue_id) { - q_conf.schedule_type = RTE_SCHED_TYPE_ATOMIC; + q_conf.event_queue_cfg = 0; + slot = queue % (nb_stages + 1); + if (slot == nb_stages) { + q_conf.schedule_type = RTE_SCHED_TYPE_ATOMIC; + if (!t->internal_port) { q_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK; - } else { - q_conf.schedule_type = - opt->sched_type_list[slot]; - queue_arr[nb_worker_queues] = queue; - nb_worker_queues++; } + tx_evqueue_id[prod++] = queue; + } else { + q_conf.schedule_type = opt->sched_type_list[slot]; + queue_arr[nb_worker_queues] = queue; + nb_worker_queues++; } ret = rte_event_queue_setup(opt->dev_id, queue, &q_conf); @@ -407,19 +391,11 @@ pipeline_queue_eventdev_setup(struct evt_test *test, struct evt_options *opt) .new_event_threshold = info.max_num_events, }; - /* - * If tx is multi thread safe then allow workers to do Tx else use Tx - * service to Tx packets. - */ - if (t->mt_unsafe) { + if (!t->internal_port) { ret = pipeline_event_port_setup(test, opt, queue_arr, nb_worker_queues, p_conf); if (ret) return ret; - - ret = pipeline_event_tx_service_setup(test, opt, tx_evqueue_id, - nb_ports - 1, p_conf); - } else ret = pipeline_event_port_setup(test, opt, NULL, nb_queues, p_conf); @@ -431,7 +407,6 @@ pipeline_queue_eventdev_setup(struct evt_test *test, struct evt_options *opt) * * eth_dev_count = 2, nb_stages = 2. * - * Multi thread safe : * queues = 6 * stride = 3 * @@ -439,21 +414,14 @@ pipeline_queue_eventdev_setup(struct evt_test *test, struct evt_options *opt) * eth0 -> q0 -> q1 -> (q2->tx) * eth1 -> q3 -> q4 -> (q5->tx) * - * q2, q5 configured as ATOMIC - * - * Multi thread unsafe : - * queues = 5 - * stride = 2 - * - * event queue pipelines: - * eth0 -> q0 -> q1 - * } (q4->tx) Tx service - * eth1 -> q2 -> q3 + * q2, q5 configured as ATOMIC | SINGLE_LINK * - * q4 configured as SINGLE_LINK|ATOMIC */ - ret = pipeline_event_rx_adapter_setup(opt, - t->mt_unsafe ? nb_stages : nb_stages + 1, p_conf); + ret = pipeline_event_rx_adapter_setup(opt, nb_stages + 1, p_conf); + if (ret) + return ret; + + ret = pipeline_event_tx_adapter_setup(opt, p_conf); if (ret) return ret; @@ -467,12 +435,60 @@ pipeline_queue_eventdev_setup(struct evt_test *test, struct evt_options *opt) } } + /* Connect the tx_evqueue_id to the Tx adapter port */ + if (!t->internal_port) { + RTE_ETH_FOREACH_DEV(prod) { + ret = rte_event_eth_tx_adapter_event_port_get(prod, + &tx_evport_id); + if (ret) { + evt_err("Unable to get Tx adptr[%d] evprt[%d]", + prod, tx_evport_id); + return ret; + } + + if (rte_event_port_link(opt->dev_id, tx_evport_id, + &tx_evqueue_id[prod], + NULL, 1) != 1) { + evt_err("Unable to link Tx adptr[%d] evprt[%d]", + prod, tx_evport_id); + return ret; + } + } + } + + RTE_ETH_FOREACH_DEV(prod) { + ret = rte_eth_dev_start(prod); + if (ret) { + evt_err("Ethernet dev [%d] failed to start." + " Using synthetic producer", prod); + return ret; + } + + } + ret = rte_event_dev_start(opt->dev_id); if (ret) { evt_err("failed to start eventdev %d", opt->dev_id); return ret; } + RTE_ETH_FOREACH_DEV(prod) { + ret = rte_event_eth_rx_adapter_start(prod); + if (ret) { + evt_err("Rx adapter[%d] start failed", prod); + return ret; + } + + ret = rte_event_eth_tx_adapter_start(prod); + if (ret) { + evt_err("Tx adapter[%d] start failed", prod); + return ret; + } + } + + memcpy(t->tx_evqueue_id, tx_evqueue_id, sizeof(uint8_t) * + RTE_MAX_ETHPORTS); + return 0; } diff --git a/app/test-pmd/Makefile b/app/test-pmd/Makefile index 2b4d604b..d5258eae 100644 --- a/app/test-pmd/Makefile +++ b/app/test-pmd/Makefile @@ -33,8 +33,10 @@ SRCS-y += rxonly.c SRCS-y += txonly.c SRCS-y += csumonly.c SRCS-y += icmpecho.c +SRCS-y += noisy_vnf.c SRCS-$(CONFIG_RTE_LIBRTE_IEEE1588) += ieee1588fwd.c SRCS-$(CONFIG_RTE_LIBRTE_BPF) += bpf_cmd.c +SRCS-y += util.c ifeq ($(CONFIG_RTE_LIBRTE_PMD_SOFTNIC), y) SRCS-y += softnicfwd.c @@ -70,8 +72,6 @@ endif endif -CFLAGS_cmdline.o := -D_GNU_SOURCE - include $(RTE_SDK)/mk/rte.app.mk endif diff --git a/app/test-pmd/cmdline.c b/app/test-pmd/cmdline.c index 589121d6..1050fde9 100644 --- a/app/test-pmd/cmdline.c +++ b/app/test-pmd/cmdline.c @@ -167,7 +167,7 @@ static void cmd_help_long_parsed(void *parsed_result, "Display:\n" "--------\n\n" - "show port (info|stats|xstats|fdir|stat_qmap|dcb_tc|cap) (port_id|all)\n" + "show port (info|stats|summary|xstats|fdir|stat_qmap|dcb_tc|cap) (port_id|all)\n" " Display information for port_id, or all.\n\n" "show port X rss reta (size) (mask0,mask1,...)\n" @@ -175,11 +175,8 @@ static void cmd_help_long_parsed(void *parsed_result, " by masks on port X. size is used to indicate the" " hardware supported reta size\n\n" - "show port rss-hash ipv4|ipv4-frag|ipv4-tcp|ipv4-udp|" - "ipv4-sctp|ipv4-other|ipv6|ipv6-frag|ipv6-tcp|ipv6-udp|ipv6-sctp|" - "ipv6-other|l2-payload|ipv6-ex|ipv6-tcp-ex|ipv6-udp-ex [key]\n" - " Display the RSS hash functions and RSS hash key" - " of port X\n\n" + "show port (port_id) rss-hash [key]\n" + " Display the RSS hash functions and RSS hash key of port\n\n" "clear port (info|stats|xstats|fdir|stat_qmap) (port_id|all)\n" " Clear information for port_id, or all.\n\n" @@ -283,6 +280,9 @@ static void cmd_help_long_parsed(void *parsed_result, "set portlist (x[,y]*)\n" " Set the list of forwarding ports.\n\n" + "set port setup on (iterator|event)\n" + " Select how attached port is retrieved for setup.\n\n" + "set tx loopback (port_id) (on|off)\n" " Enable or disable tx loopback.\n\n" @@ -397,12 +397,13 @@ static void cmd_help_long_parsed(void *parsed_result, " Disable hardware insertion of a VLAN header in" " packets sent on a port.\n\n" - "csum set (ip|udp|tcp|sctp|outer-ip) (hw|sw) (port_id)\n" + "csum set (ip|udp|tcp|sctp|outer-ip|outer-udp) (hw|sw) (port_id)\n" " Select hardware or software calculation of the" " checksum when transmitting a packet using the" " csum forward engine.\n" " ip|udp|tcp|sctp always concern the inner layer.\n" " outer-ip concerns the outer IP layer in" + " outer-udp concerns the outer UDP layer in" " case the packet is recognized as a tunnel packet by" " the forward engine (vxlan, gre and ipip are supported)\n" " Please check the NIC datasheet for HW limits.\n\n" @@ -883,6 +884,10 @@ static void cmd_help_long_parsed(void *parsed_result, " Start/stop a rx/tx queue of port X. Only take effect" " when port X is started\n\n" + "port (port_id) (rxq|txq) (queue_id) deferred_start (on|off)\n" + " Switch on/off a deferred start of port X rx/tx queue. Only" + " take effect when port X is stopped.\n\n" + "port (port_id) (rxq|txq) (queue_id) setup\n" " Setup a rx/tx queue of port X.\n\n" @@ -1247,6 +1252,59 @@ cmdline_parse_inst_t cmd_operate_specific_port = { }, }; +/* *** enable port setup (after attach) via iterator or event *** */ +struct cmd_set_port_setup_on_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t port; + cmdline_fixed_string_t setup; + cmdline_fixed_string_t on; + cmdline_fixed_string_t mode; +}; + +static void cmd_set_port_setup_on_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_set_port_setup_on_result *res = parsed_result; + + if (strcmp(res->mode, "event") == 0) + setup_on_probe_event = true; + else if (strcmp(res->mode, "iterator") == 0) + setup_on_probe_event = false; + else + printf("Unknown mode\n"); +} + +cmdline_parse_token_string_t cmd_set_port_setup_on_set = + TOKEN_STRING_INITIALIZER(struct cmd_set_port_setup_on_result, + set, "set"); +cmdline_parse_token_string_t cmd_set_port_setup_on_port = + TOKEN_STRING_INITIALIZER(struct cmd_set_port_setup_on_result, + port, "port"); +cmdline_parse_token_string_t cmd_set_port_setup_on_setup = + TOKEN_STRING_INITIALIZER(struct cmd_set_port_setup_on_result, + setup, "setup"); +cmdline_parse_token_string_t cmd_set_port_setup_on_on = + TOKEN_STRING_INITIALIZER(struct cmd_set_port_setup_on_result, + on, "on"); +cmdline_parse_token_string_t cmd_set_port_setup_on_mode = + TOKEN_STRING_INITIALIZER(struct cmd_set_port_setup_on_result, + mode, "iterator#event"); + +cmdline_parse_inst_t cmd_set_port_setup_on = { + .f = cmd_set_port_setup_on_parsed, + .data = NULL, + .help_str = "set port setup on iterator|event", + .tokens = { + (void *)&cmd_set_port_setup_on_set, + (void *)&cmd_set_port_setup_on_port, + (void *)&cmd_set_port_setup_on_setup, + (void *)&cmd_set_port_setup_on_on, + (void *)&cmd_set_port_setup_on_mode, + NULL, + }, +}; + /* *** attach a specified port *** */ struct cmd_operate_attach_port_result { cmdline_fixed_string_t port; @@ -1303,7 +1361,7 @@ static void cmd_operate_detach_port_parsed(void *parsed_result, struct cmd_operate_detach_port_result *res = parsed_result; if (!strcmp(res->keyword, "detach")) - detach_port(res->port_id); + detach_port_device(res->port_id); else printf("Unknown parameter\n"); } @@ -1898,11 +1956,9 @@ cmd_config_rx_mode_flag_parsed(void *parsed_result, rx_offloads = port->dev_conf.rxmode.offloads; if (!strcmp(res->name, "crc-strip")) { if (!strcmp(res->value, "on")) { - rx_offloads |= DEV_RX_OFFLOAD_CRC_STRIP; rx_offloads &= ~DEV_RX_OFFLOAD_KEEP_CRC; } else if (!strcmp(res->value, "off")) { rx_offloads |= DEV_RX_OFFLOAD_KEEP_CRC; - rx_offloads &= ~DEV_RX_OFFLOAD_CRC_STRIP; } else { printf("Unknown parameter\n"); return; @@ -2441,6 +2497,92 @@ cmdline_parse_inst_t cmd_config_rxtx_queue = { }, }; +/* *** configure port rxq/txq deferred start on/off *** */ +struct cmd_config_deferred_start_rxtx_queue { + cmdline_fixed_string_t port; + portid_t port_id; + cmdline_fixed_string_t rxtxq; + uint16_t qid; + cmdline_fixed_string_t opname; + cmdline_fixed_string_t state; +}; + +static void +cmd_config_deferred_start_rxtx_queue_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_config_deferred_start_rxtx_queue *res = parsed_result; + struct rte_port *port; + uint8_t isrx; + uint8_t ison; + uint8_t needreconfig = 0; + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + + if (port_is_started(res->port_id) != 0) { + printf("Please stop port %u first\n", res->port_id); + return; + } + + port = &ports[res->port_id]; + + isrx = !strcmp(res->rxtxq, "rxq"); + + if (isrx && rx_queue_id_is_invalid(res->qid)) + return; + else if (!isrx && tx_queue_id_is_invalid(res->qid)) + return; + + ison = !strcmp(res->state, "on"); + + if (isrx && port->rx_conf[res->qid].rx_deferred_start != ison) { + port->rx_conf[res->qid].rx_deferred_start = ison; + needreconfig = 1; + } else if (!isrx && port->tx_conf[res->qid].tx_deferred_start != ison) { + port->tx_conf[res->qid].tx_deferred_start = ison; + needreconfig = 1; + } + + if (needreconfig) + cmd_reconfig_device_queue(res->port_id, 0, 1); +} + +cmdline_parse_token_string_t cmd_config_deferred_start_rxtx_queue_port = + TOKEN_STRING_INITIALIZER(struct cmd_config_deferred_start_rxtx_queue, + port, "port"); +cmdline_parse_token_num_t cmd_config_deferred_start_rxtx_queue_port_id = + TOKEN_NUM_INITIALIZER(struct cmd_config_deferred_start_rxtx_queue, + port_id, UINT16); +cmdline_parse_token_string_t cmd_config_deferred_start_rxtx_queue_rxtxq = + TOKEN_STRING_INITIALIZER(struct cmd_config_deferred_start_rxtx_queue, + rxtxq, "rxq#txq"); +cmdline_parse_token_num_t cmd_config_deferred_start_rxtx_queue_qid = + TOKEN_NUM_INITIALIZER(struct cmd_config_deferred_start_rxtx_queue, + qid, UINT16); +cmdline_parse_token_string_t cmd_config_deferred_start_rxtx_queue_opname = + TOKEN_STRING_INITIALIZER(struct cmd_config_deferred_start_rxtx_queue, + opname, "deferred_start"); +cmdline_parse_token_string_t cmd_config_deferred_start_rxtx_queue_state = + TOKEN_STRING_INITIALIZER(struct cmd_config_deferred_start_rxtx_queue, + state, "on#off"); + +cmdline_parse_inst_t cmd_config_deferred_start_rxtx_queue = { + .f = cmd_config_deferred_start_rxtx_queue_parsed, + .data = NULL, + .help_str = "port rxq|txq deferred_start on|off", + .tokens = { + (void *)&cmd_config_deferred_start_rxtx_queue_port, + (void *)&cmd_config_deferred_start_rxtx_queue_port_id, + (void *)&cmd_config_deferred_start_rxtx_queue_rxtxq, + (void *)&cmd_config_deferred_start_rxtx_queue_qid, + (void *)&cmd_config_deferred_start_rxtx_queue_opname, + (void *)&cmd_config_deferred_start_rxtx_queue_state, + NULL, + }, +}; + /* *** configure port rxq/txq setup *** */ struct cmd_setup_rxtx_queue { cmdline_fixed_string_t port; @@ -2816,8 +2958,7 @@ static void cmd_showport_rss_hash_parsed(void *parsed_result, { struct cmd_showport_rss_hash *res = parsed_result; - port_rss_hash_conf_show(res->port_id, res->rss_type, - show_rss_key != NULL); + port_rss_hash_conf_show(res->port_id, show_rss_key != NULL); } cmdline_parse_token_string_t cmd_showport_rss_hash_show = @@ -2829,28 +2970,18 @@ cmdline_parse_token_num_t cmd_showport_rss_hash_port_id = cmdline_parse_token_string_t cmd_showport_rss_hash_rss_hash = TOKEN_STRING_INITIALIZER(struct cmd_showport_rss_hash, rss_hash, "rss-hash"); -cmdline_parse_token_string_t cmd_showport_rss_hash_rss_hash_info = - TOKEN_STRING_INITIALIZER(struct cmd_showport_rss_hash, rss_type, - "ipv4#ipv4-frag#ipv4-tcp#ipv4-udp#ipv4-sctp#" - "ipv4-other#ipv6#ipv6-frag#ipv6-tcp#ipv6-udp#" - "ipv6-sctp#ipv6-other#l2-payload#ipv6-ex#" - "ipv6-tcp-ex#ipv6-udp-ex"); cmdline_parse_token_string_t cmd_showport_rss_hash_rss_key = TOKEN_STRING_INITIALIZER(struct cmd_showport_rss_hash, key, "key"); cmdline_parse_inst_t cmd_showport_rss_hash = { .f = cmd_showport_rss_hash_parsed, .data = NULL, - .help_str = "show port rss-hash " - "ipv4|ipv4-frag|ipv4-tcp|ipv4-udp|ipv4-sctp|ipv4-other|" - "ipv6|ipv6-frag|ipv6-tcp|ipv6-udp|ipv6-sctp|ipv6-other|" - "l2-payload|ipv6-ex|ipv6-tcp-ex|ipv6-udp-ex", + .help_str = "show port rss-hash", .tokens = { (void *)&cmd_showport_rss_hash_show, (void *)&cmd_showport_rss_hash_port, (void *)&cmd_showport_rss_hash_port_id, (void *)&cmd_showport_rss_hash_rss_hash, - (void *)&cmd_showport_rss_hash_rss_hash_info, NULL, }, }; @@ -2858,16 +2989,12 @@ cmdline_parse_inst_t cmd_showport_rss_hash = { cmdline_parse_inst_t cmd_showport_rss_hash_key = { .f = cmd_showport_rss_hash_parsed, .data = (void *)1, - .help_str = "show port rss-hash " - "ipv4|ipv4-frag|ipv4-tcp|ipv4-udp|ipv4-sctp|ipv4-other|" - "ipv6|ipv6-frag|ipv6-tcp|ipv6-udp|ipv6-sctp|ipv6-other|" - "l2-payload|ipv6-ex|ipv6-tcp-ex|ipv6-udp-ex key", + .help_str = "show port rss-hash key", .tokens = { (void *)&cmd_showport_rss_hash_show, (void *)&cmd_showport_rss_hash_port, (void *)&cmd_showport_rss_hash_port_id, (void *)&cmd_showport_rss_hash_rss_hash, - (void *)&cmd_showport_rss_hash_rss_hash_info, (void *)&cmd_showport_rss_hash_rss_key, NULL, }, @@ -4089,6 +4216,8 @@ csum_show(int port_id) (tx_offloads & DEV_TX_OFFLOAD_SCTP_CKSUM) ? "hw" : "sw"); printf("Outer-Ip checksum offload is %s\n", (tx_offloads & DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM) ? "hw" : "sw"); + printf("Outer-Udp checksum offload is %s\n", + (tx_offloads & DEV_TX_OFFLOAD_OUTER_UDP_CKSUM) ? "hw" : "sw"); /* display warnings if configuration is not supported by the NIC */ rte_eth_dev_info_get(port_id, &dev_info); @@ -4117,6 +4246,12 @@ csum_show(int port_id) printf("Warning: hardware outer IP checksum enabled but not " "supported by port %d\n", port_id); } + if ((tx_offloads & DEV_TX_OFFLOAD_OUTER_UDP_CKSUM) && + (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_OUTER_UDP_CKSUM) + == 0) { + printf("Warning: hardware outer UDP checksum enabled but not " + "supported by port %d\n", port_id); + } } static void @@ -4185,6 +4320,15 @@ cmd_csum_parsed(void *parsed_result, printf("Outer IP checksum offload is not " "supported by port %u\n", res->port_id); } + } else if (!strcmp(res->proto, "outer-udp")) { + if (hw == 0 || (dev_info.tx_offload_capa & + DEV_TX_OFFLOAD_OUTER_UDP_CKSUM)) { + csum_offloads |= + DEV_TX_OFFLOAD_OUTER_UDP_CKSUM; + } else { + printf("Outer UDP checksum offload is not " + "supported by port %u\n", res->port_id); + } } if (hw) { @@ -4208,7 +4352,7 @@ cmdline_parse_token_string_t cmd_csum_mode = mode, "set"); cmdline_parse_token_string_t cmd_csum_proto = TOKEN_STRING_INITIALIZER(struct cmd_csum_result, - proto, "ip#tcp#udp#sctp#outer-ip"); + proto, "ip#tcp#udp#sctp#outer-ip#outer-udp"); cmdline_parse_token_string_t cmd_csum_hwsw = TOKEN_STRING_INITIALIZER(struct cmd_csum_result, hwsw, "hw#sw"); @@ -4219,7 +4363,7 @@ cmdline_parse_token_num_t cmd_csum_portid = cmdline_parse_inst_t cmd_csum_set = { .f = cmd_csum_parsed, .data = NULL, - .help_str = "csum set ip|tcp|udp|sctp|outer-ip hw|sw : " + .help_str = "csum set ip|tcp|udp|sctp|outer-ip|outer-udp hw|sw : " "Enable/Disable hardware calculation of L3/L4 checksum when " "using csum forward engine", .tokens = { @@ -4279,7 +4423,7 @@ cmdline_parse_token_string_t cmd_csum_tunnel_csum = csum, "csum"); cmdline_parse_token_string_t cmd_csum_tunnel_parse = TOKEN_STRING_INITIALIZER(struct cmd_csum_tunnel_result, - parse, "parse_tunnel"); + parse, "parse-tunnel"); cmdline_parse_token_string_t cmd_csum_tunnel_onoff = TOKEN_STRING_INITIALIZER(struct cmd_csum_tunnel_result, onoff, "on#off"); @@ -4290,7 +4434,7 @@ cmdline_parse_token_num_t cmd_csum_tunnel_portid = cmdline_parse_inst_t cmd_csum_tunnel = { .f = cmd_csum_tunnel_parsed, .data = NULL, - .help_str = "csum parse_tunnel on|off : " + .help_str = "csum parse-tunnel on|off : " "Enable/Disable parsing of tunnels for csum engine", .tokens = { (void *)&cmd_csum_tunnel_csum, @@ -7073,6 +7217,11 @@ static void cmd_showportall_parsed(void *parsed_result, } else if (!strcmp(res->what, "info")) RTE_ETH_FOREACH_DEV(i) port_infos_display(i); + else if (!strcmp(res->what, "summary")) { + port_summary_header_display(); + RTE_ETH_FOREACH_DEV(i) + port_summary_display(i); + } else if (!strcmp(res->what, "stats")) RTE_ETH_FOREACH_DEV(i) nic_stats_display(i); @@ -7100,14 +7249,14 @@ cmdline_parse_token_string_t cmd_showportall_port = TOKEN_STRING_INITIALIZER(struct cmd_showportall_result, port, "port"); cmdline_parse_token_string_t cmd_showportall_what = TOKEN_STRING_INITIALIZER(struct cmd_showportall_result, what, - "info#stats#xstats#fdir#stat_qmap#dcb_tc#cap"); + "info#summary#stats#xstats#fdir#stat_qmap#dcb_tc#cap"); cmdline_parse_token_string_t cmd_showportall_all = TOKEN_STRING_INITIALIZER(struct cmd_showportall_result, all, "all"); cmdline_parse_inst_t cmd_showportall = { .f = cmd_showportall_parsed, .data = NULL, .help_str = "show|clear port " - "info|stats|xstats|fdir|stat_qmap|dcb_tc|cap all", + "info|summary|stats|xstats|fdir|stat_qmap|dcb_tc|cap all", .tokens = { (void *)&cmd_showportall_show, (void *)&cmd_showportall_port, @@ -7137,6 +7286,10 @@ static void cmd_showport_parsed(void *parsed_result, nic_xstats_clear(res->portnum); } else if (!strcmp(res->what, "info")) port_infos_display(res->portnum); + else if (!strcmp(res->what, "summary")) { + port_summary_header_display(); + port_summary_display(res->portnum); + } else if (!strcmp(res->what, "stats")) nic_stats_display(res->portnum); else if (!strcmp(res->what, "xstats")) @@ -7158,7 +7311,7 @@ cmdline_parse_token_string_t cmd_showport_port = TOKEN_STRING_INITIALIZER(struct cmd_showport_result, port, "port"); cmdline_parse_token_string_t cmd_showport_what = TOKEN_STRING_INITIALIZER(struct cmd_showport_result, what, - "info#stats#xstats#fdir#stat_qmap#dcb_tc#cap"); + "info#summary#stats#xstats#fdir#stat_qmap#dcb_tc#cap"); cmdline_parse_token_num_t cmd_showport_portnum = TOKEN_NUM_INITIALIZER(struct cmd_showport_result, portnum, UINT16); @@ -7166,7 +7319,7 @@ cmdline_parse_inst_t cmd_showport = { .f = cmd_showport_parsed, .data = NULL, .help_str = "show|clear port " - "info|stats|xstats|fdir|stat_qmap|dcb_tc|cap " + "info|summary|stats|xstats|fdir|stat_qmap|dcb_tc|cap " "", .tokens = { (void *)&cmd_showport_show, @@ -7573,7 +7726,6 @@ static void cmd_quit_parsed(__attribute__((unused)) void *parsed_result, struct cmdline *cl, __attribute__((unused)) void *data) { - pmd_test_exit(); cmdline_quit(cl); } @@ -15186,6 +15338,631 @@ cmdline_parse_inst_t cmd_set_nvgre_with_vlan = { }, }; +/** Set L2 encapsulation details */ +struct cmd_set_l2_encap_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t l2_encap; + cmdline_fixed_string_t pos_token; + cmdline_fixed_string_t ip_version; + uint32_t vlan_present:1; + uint16_t tci; + struct ether_addr eth_src; + struct ether_addr eth_dst; +}; + +cmdline_parse_token_string_t cmd_set_l2_encap_set = + TOKEN_STRING_INITIALIZER(struct cmd_set_l2_encap_result, set, "set"); +cmdline_parse_token_string_t cmd_set_l2_encap_l2_encap = + TOKEN_STRING_INITIALIZER(struct cmd_set_l2_encap_result, l2_encap, "l2_encap"); +cmdline_parse_token_string_t cmd_set_l2_encap_l2_encap_with_vlan = + TOKEN_STRING_INITIALIZER(struct cmd_set_l2_encap_result, l2_encap, + "l2_encap-with-vlan"); +cmdline_parse_token_string_t cmd_set_l2_encap_ip_version = + TOKEN_STRING_INITIALIZER(struct cmd_set_l2_encap_result, pos_token, + "ip-version"); +cmdline_parse_token_string_t cmd_set_l2_encap_ip_version_value = + TOKEN_STRING_INITIALIZER(struct cmd_set_l2_encap_result, ip_version, + "ipv4#ipv6"); +cmdline_parse_token_string_t cmd_set_l2_encap_vlan = + TOKEN_STRING_INITIALIZER(struct cmd_set_l2_encap_result, pos_token, + "vlan-tci"); +cmdline_parse_token_num_t cmd_set_l2_encap_vlan_value = + TOKEN_NUM_INITIALIZER(struct cmd_set_l2_encap_result, tci, UINT16); +cmdline_parse_token_string_t cmd_set_l2_encap_eth_src = + TOKEN_STRING_INITIALIZER(struct cmd_set_l2_encap_result, pos_token, + "eth-src"); +cmdline_parse_token_etheraddr_t cmd_set_l2_encap_eth_src_value = + TOKEN_ETHERADDR_INITIALIZER(struct cmd_set_l2_encap_result, eth_src); +cmdline_parse_token_string_t cmd_set_l2_encap_eth_dst = + TOKEN_STRING_INITIALIZER(struct cmd_set_l2_encap_result, pos_token, + "eth-dst"); +cmdline_parse_token_etheraddr_t cmd_set_l2_encap_eth_dst_value = + TOKEN_ETHERADDR_INITIALIZER(struct cmd_set_l2_encap_result, eth_dst); + +static void cmd_set_l2_encap_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_set_l2_encap_result *res = parsed_result; + + if (strcmp(res->l2_encap, "l2_encap") == 0) + l2_encap_conf.select_vlan = 0; + else if (strcmp(res->l2_encap, "l2_encap-with-vlan") == 0) + l2_encap_conf.select_vlan = 1; + if (strcmp(res->ip_version, "ipv4") == 0) + l2_encap_conf.select_ipv4 = 1; + else if (strcmp(res->ip_version, "ipv6") == 0) + l2_encap_conf.select_ipv4 = 0; + else + return; + if (l2_encap_conf.select_vlan) + l2_encap_conf.vlan_tci = rte_cpu_to_be_16(res->tci); + rte_memcpy(l2_encap_conf.eth_src, res->eth_src.addr_bytes, + ETHER_ADDR_LEN); + rte_memcpy(l2_encap_conf.eth_dst, res->eth_dst.addr_bytes, + ETHER_ADDR_LEN); +} + +cmdline_parse_inst_t cmd_set_l2_encap = { + .f = cmd_set_l2_encap_parsed, + .data = NULL, + .help_str = "set l2_encap ip-version ipv4|ipv6" + " eth-src eth-dst ", + .tokens = { + (void *)&cmd_set_l2_encap_set, + (void *)&cmd_set_l2_encap_l2_encap, + (void *)&cmd_set_l2_encap_ip_version, + (void *)&cmd_set_l2_encap_ip_version_value, + (void *)&cmd_set_l2_encap_eth_src, + (void *)&cmd_set_l2_encap_eth_src_value, + (void *)&cmd_set_l2_encap_eth_dst, + (void *)&cmd_set_l2_encap_eth_dst_value, + NULL, + }, +}; + +cmdline_parse_inst_t cmd_set_l2_encap_with_vlan = { + .f = cmd_set_l2_encap_parsed, + .data = NULL, + .help_str = "set l2_encap-with-vlan ip-version ipv4|ipv6" + " vlan-tci eth-src eth-dst ", + .tokens = { + (void *)&cmd_set_l2_encap_set, + (void *)&cmd_set_l2_encap_l2_encap_with_vlan, + (void *)&cmd_set_l2_encap_ip_version, + (void *)&cmd_set_l2_encap_ip_version_value, + (void *)&cmd_set_l2_encap_vlan, + (void *)&cmd_set_l2_encap_vlan_value, + (void *)&cmd_set_l2_encap_eth_src, + (void *)&cmd_set_l2_encap_eth_src_value, + (void *)&cmd_set_l2_encap_eth_dst, + (void *)&cmd_set_l2_encap_eth_dst_value, + NULL, + }, +}; + +/** Set L2 decapsulation details */ +struct cmd_set_l2_decap_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t l2_decap; + cmdline_fixed_string_t pos_token; + uint32_t vlan_present:1; +}; + +cmdline_parse_token_string_t cmd_set_l2_decap_set = + TOKEN_STRING_INITIALIZER(struct cmd_set_l2_decap_result, set, "set"); +cmdline_parse_token_string_t cmd_set_l2_decap_l2_decap = + TOKEN_STRING_INITIALIZER(struct cmd_set_l2_decap_result, l2_decap, + "l2_decap"); +cmdline_parse_token_string_t cmd_set_l2_decap_l2_decap_with_vlan = + TOKEN_STRING_INITIALIZER(struct cmd_set_l2_decap_result, l2_decap, + "l2_decap-with-vlan"); + +static void cmd_set_l2_decap_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_set_l2_decap_result *res = parsed_result; + + if (strcmp(res->l2_decap, "l2_decap") == 0) + l2_decap_conf.select_vlan = 0; + else if (strcmp(res->l2_decap, "l2_decap-with-vlan") == 0) + l2_decap_conf.select_vlan = 1; +} + +cmdline_parse_inst_t cmd_set_l2_decap = { + .f = cmd_set_l2_decap_parsed, + .data = NULL, + .help_str = "set l2_decap", + .tokens = { + (void *)&cmd_set_l2_decap_set, + (void *)&cmd_set_l2_decap_l2_decap, + NULL, + }, +}; + +cmdline_parse_inst_t cmd_set_l2_decap_with_vlan = { + .f = cmd_set_l2_decap_parsed, + .data = NULL, + .help_str = "set l2_decap-with-vlan", + .tokens = { + (void *)&cmd_set_l2_decap_set, + (void *)&cmd_set_l2_decap_l2_decap_with_vlan, + NULL, + }, +}; + +/** Set MPLSoGRE encapsulation details */ +struct cmd_set_mplsogre_encap_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t mplsogre; + cmdline_fixed_string_t pos_token; + cmdline_fixed_string_t ip_version; + uint32_t vlan_present:1; + uint32_t label; + cmdline_ipaddr_t ip_src; + cmdline_ipaddr_t ip_dst; + uint16_t tci; + struct ether_addr eth_src; + struct ether_addr eth_dst; +}; + +cmdline_parse_token_string_t cmd_set_mplsogre_encap_set = + TOKEN_STRING_INITIALIZER(struct cmd_set_mplsogre_encap_result, set, + "set"); +cmdline_parse_token_string_t cmd_set_mplsogre_encap_mplsogre_encap = + TOKEN_STRING_INITIALIZER(struct cmd_set_mplsogre_encap_result, mplsogre, + "mplsogre_encap"); +cmdline_parse_token_string_t cmd_set_mplsogre_encap_mplsogre_encap_with_vlan = + TOKEN_STRING_INITIALIZER(struct cmd_set_mplsogre_encap_result, + mplsogre, "mplsogre_encap-with-vlan"); +cmdline_parse_token_string_t cmd_set_mplsogre_encap_ip_version = + TOKEN_STRING_INITIALIZER(struct cmd_set_mplsogre_encap_result, + pos_token, "ip-version"); +cmdline_parse_token_string_t cmd_set_mplsogre_encap_ip_version_value = + TOKEN_STRING_INITIALIZER(struct cmd_set_mplsogre_encap_result, + ip_version, "ipv4#ipv6"); +cmdline_parse_token_string_t cmd_set_mplsogre_encap_label = + TOKEN_STRING_INITIALIZER(struct cmd_set_mplsogre_encap_result, + pos_token, "label"); +cmdline_parse_token_num_t cmd_set_mplsogre_encap_label_value = + TOKEN_NUM_INITIALIZER(struct cmd_set_mplsogre_encap_result, label, + UINT32); +cmdline_parse_token_string_t cmd_set_mplsogre_encap_ip_src = + TOKEN_STRING_INITIALIZER(struct cmd_set_mplsogre_encap_result, + pos_token, "ip-src"); +cmdline_parse_token_ipaddr_t cmd_set_mplsogre_encap_ip_src_value = + TOKEN_IPADDR_INITIALIZER(struct cmd_set_mplsogre_encap_result, ip_src); +cmdline_parse_token_string_t cmd_set_mplsogre_encap_ip_dst = + TOKEN_STRING_INITIALIZER(struct cmd_set_mplsogre_encap_result, + pos_token, "ip-dst"); +cmdline_parse_token_ipaddr_t cmd_set_mplsogre_encap_ip_dst_value = + TOKEN_IPADDR_INITIALIZER(struct cmd_set_mplsogre_encap_result, ip_dst); +cmdline_parse_token_string_t cmd_set_mplsogre_encap_vlan = + TOKEN_STRING_INITIALIZER(struct cmd_set_mplsogre_encap_result, + pos_token, "vlan-tci"); +cmdline_parse_token_num_t cmd_set_mplsogre_encap_vlan_value = + TOKEN_NUM_INITIALIZER(struct cmd_set_mplsogre_encap_result, tci, + UINT16); +cmdline_parse_token_string_t cmd_set_mplsogre_encap_eth_src = + TOKEN_STRING_INITIALIZER(struct cmd_set_mplsogre_encap_result, + pos_token, "eth-src"); +cmdline_parse_token_etheraddr_t cmd_set_mplsogre_encap_eth_src_value = + TOKEN_ETHERADDR_INITIALIZER(struct cmd_set_mplsogre_encap_result, + eth_src); +cmdline_parse_token_string_t cmd_set_mplsogre_encap_eth_dst = + TOKEN_STRING_INITIALIZER(struct cmd_set_mplsogre_encap_result, + pos_token, "eth-dst"); +cmdline_parse_token_etheraddr_t cmd_set_mplsogre_encap_eth_dst_value = + TOKEN_ETHERADDR_INITIALIZER(struct cmd_set_mplsogre_encap_result, + eth_dst); + +static void cmd_set_mplsogre_encap_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_set_mplsogre_encap_result *res = parsed_result; + union { + uint32_t mplsogre_label; + uint8_t label[3]; + } id = { + .mplsogre_label = + rte_cpu_to_be_32(res->label) & RTE_BE32(0x00ffffff), + }; + + if (strcmp(res->mplsogre, "mplsogre_encap") == 0) + mplsogre_encap_conf.select_vlan = 0; + else if (strcmp(res->mplsogre, "mplsogre_encap-with-vlan") == 0) + mplsogre_encap_conf.select_vlan = 1; + if (strcmp(res->ip_version, "ipv4") == 0) + mplsogre_encap_conf.select_ipv4 = 1; + else if (strcmp(res->ip_version, "ipv6") == 0) + mplsogre_encap_conf.select_ipv4 = 0; + else + return; + rte_memcpy(mplsogre_encap_conf.label, &id.label[1], 3); + if (mplsogre_encap_conf.select_ipv4) { + IPV4_ADDR_TO_UINT(res->ip_src, mplsogre_encap_conf.ipv4_src); + IPV4_ADDR_TO_UINT(res->ip_dst, mplsogre_encap_conf.ipv4_dst); + } else { + IPV6_ADDR_TO_ARRAY(res->ip_src, mplsogre_encap_conf.ipv6_src); + IPV6_ADDR_TO_ARRAY(res->ip_dst, mplsogre_encap_conf.ipv6_dst); + } + if (mplsogre_encap_conf.select_vlan) + mplsogre_encap_conf.vlan_tci = rte_cpu_to_be_16(res->tci); + rte_memcpy(mplsogre_encap_conf.eth_src, res->eth_src.addr_bytes, + ETHER_ADDR_LEN); + rte_memcpy(mplsogre_encap_conf.eth_dst, res->eth_dst.addr_bytes, + ETHER_ADDR_LEN); +} + +cmdline_parse_inst_t cmd_set_mplsogre_encap = { + .f = cmd_set_mplsogre_encap_parsed, + .data = NULL, + .help_str = "set mplsogre_encap ip-version ipv4|ipv6 label