From fb731e12fc85bf225e8917fe068fcc357f9a2158 Mon Sep 17 00:00:00 2001 From: pmikus Date: Wed, 26 Jul 2023 13:49:46 +0000 Subject: [PATCH] fix(core): QAT initialization Signed-off-by: pmikus Change-Id: I28af64b7d39a97bbb84f9d987a3f5d3a2d02701b --- fdio.infra.ansible/roles/intel/tasks/qat.yaml | 10 ++ resources/libraries/python/Constants.py | 2 +- resources/libraries/python/DUTSetup.py | 85 +---------------- resources/libraries/python/QATUtil.py | 105 +++++++++++++++++++++ resources/libraries/robot/shared/default.robot | 1 + resources/libraries/robot/shared/suite_setup.robot | 1 - 6 files changed, 118 insertions(+), 86 deletions(-) create mode 100644 resources/libraries/python/QATUtil.py diff --git a/fdio.infra.ansible/roles/intel/tasks/qat.yaml b/fdio.infra.ansible/roles/intel/tasks/qat.yaml index 10dab99dc8..758106939f 100644 --- a/fdio.infra.ansible/roles/intel/tasks/qat.yaml +++ b/fdio.infra.ansible/roles/intel/tasks/qat.yaml @@ -43,5 +43,15 @@ - "make install" when: - intel_driver_extracted + tags: + - intel-inst + +- name: Load Kernel Modules By Default + ansible.builtin.lineinfile: + path: "/etc/modules" + state: "present" + line: "{{ item }}" + with_items: + - "qat_4xxx" tags: - intel-inst \ No newline at end of file diff --git a/resources/libraries/python/Constants.py b/resources/libraries/python/Constants.py index f30f5fa5e6..7bc976cca5 100644 --- a/resources/libraries/python/Constants.py +++ b/resources/libraries/python/Constants.py @@ -180,7 +180,7 @@ class Constants: QEMU_VM_IMAGE = u"/var/lib/vm/image.iso" # QEMU VM DPDK path - QEMU_VM_DPDK = u"/opt/dpdk-22.07" + QEMU_VM_DPDK = u"/opt/dpdk-23.03" # Docker container SUT image DOCKER_SUT_IMAGE_UBUNTU = u"csit_sut-ubuntu2204:local" diff --git a/resources/libraries/python/DUTSetup.py b/resources/libraries/python/DUTSetup.py index e715f48a76..a9b549a935 100644 --- a/resources/libraries/python/DUTSetup.py +++ b/resources/libraries/python/DUTSetup.py @@ -34,7 +34,7 @@ class DUTSetup: :type service: str """ if DUTSetup.running_in_container(node): - command = u"cat /var/log/vpp/vpp.log" + return else: command = ( f"journalctl --no-pager _SYSTEMD_INVOCATION_ID=$(systemctl " @@ -246,89 +246,6 @@ class DUTSetup: pids[node[u"host"]] = DUTSetup.get_pid(node, u"vpp") return pids - @staticmethod - def crypto_device_verify(node, crypto_type, numvfs, force_init=False): - """Verify if Crypto QAT device virtual functions are initialized on all - DUTs. If parameter force initialization is set to True, then try to - initialize or remove VFs on QAT. - - :param node: DUT node. - :crypto_type: Crypto device type - HW_DH895xcc, HW_C3xxx, HW_C4xxx - or HW_4xxx. - :param numvfs: Number of VFs to initialize, 0 - disable the VFs. - :param force_init: If True then try to initialize to specific value. - :type node: dict - :type crypto_type: string - :type numvfs: int - :type force_init: bool - :returns: nothing - :raises RuntimeError: If QAT VFs are not created and force init is set - to False. - """ - pci_addr = Topology.get_cryptodev(node) - sriov_numvfs = DUTSetup.get_sriov_numvfs(node, pci_addr) - - if sriov_numvfs != numvfs: - if force_init: - # QAT is not initialized and we want to initialize with numvfs - DUTSetup.crypto_device_init(node, crypto_type, numvfs) - else: - raise RuntimeError( - f"QAT device failed to create VFs on {node[u'host']}" - ) - - @staticmethod - def crypto_device_init(node, crypto_type, numvfs): - """Init Crypto QAT device virtual functions on DUT. - - :param node: DUT node. - :crypto_type: Crypto device type - HW_DH895xcc, HW_C3xxx, HW_C4xxx - or HW_4xxx. - :param numvfs: Number of VFs to initialize, 0 - disable the VFs. - :type node: dict - :type crypto_type: string - :type numvfs: int - :returns: nothing - :raises RuntimeError: If failed to stop VPP or QAT failed to initialize. - """ - if crypto_type == u"HW_DH895xcc": - kernel_mod = u"qat_dh895xcc" - kernel_drv = u"dh895xcc" - elif crypto_type == u"HW_C3xxx": - kernel_mod = u"qat_c3xxx" - kernel_drv = u"c3xxx" - elif crypto_type == u"HW_C4xxx": - kernel_mod = u"qat_c4xxx" - kernel_drv = u"c4xxx" - elif crypto_type == u"HW_4xxx": - kernel_mod = u"qat_4xxx" - kernel_drv = u"4xxx" - else: - raise RuntimeError( - f"Unsupported crypto device type on {node[u'host']}" - ) - - pci_addr = Topology.get_cryptodev(node) - - # QAT device must be re-bound to kernel driver before initialization. - DUTSetup.verify_kernel_module(node, kernel_mod, force_load=True) - - # Stop VPP to prevent deadlock. - DUTSetup.stop_service(node, Constants.VPP_UNIT) - - current_driver = DUTSetup.get_pci_dev_driver( - node, pci_addr.replace(u":", r"\:") - ) - if current_driver is not None: - DUTSetup.pci_driver_unbind(node, pci_addr) - - # Bind to kernel driver. - DUTSetup.pci_driver_bind(node, pci_addr, kernel_drv) - - # Initialize QAT VFs. - if numvfs > 0: - DUTSetup.set_sriov_numvfs(node, pci_addr, numvfs) - @staticmethod def get_virtfn_pci_addr(node, pf_pci_addr, vf_id): """Get PCI address of Virtual Function. diff --git a/resources/libraries/python/QATUtil.py b/resources/libraries/python/QATUtil.py new file mode 100644 index 0000000000..e709aad6ed --- /dev/null +++ b/resources/libraries/python/QATUtil.py @@ -0,0 +1,105 @@ +# Copyright (c) 2023 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""QAT util library.""" + +from resources.libraries.python.DUTSetup import DUTSetup +from resources.libraries.python.topology import Topology +from resources.libraries.python.VPPUtil import VPPUtil + + +class QATUtil: + """Contains methods for setting up QATs.""" + + @staticmethod + def crypto_device_verify(node, crypto_type, numvfs, force_init=False): + """Verify if Crypto QAT device virtual functions are initialized on all + DUTs. If parameter force initialization is set to True, then try to + initialize or remove VFs on QAT. + + :param node: DUT node. + :crypto_type: Crypto device type - HW_DH895xcc, HW_C3xxx, HW_C4xxx + or HW_4xxx. + :param numvfs: Number of VFs to initialize, 0 - disable the VFs. + :param force_init: If True then try to initialize to specific value. + :type node: dict + :type crypto_type: string + :type numvfs: int + :type force_init: bool + :returns: nothing + :raises RuntimeError: If QAT VFs are not created and force init is set + to False. + """ + pci_addr = Topology.get_cryptodev(node) + sriov_numvfs = DUTSetup.get_sriov_numvfs(node, pci_addr) + + if sriov_numvfs != numvfs: + if force_init: + # QAT is not initialized and we want to initialize with numvfs + QATUtil.crypto_device_init(node, crypto_type, numvfs) + else: + raise RuntimeError( + f"QAT device failed to create VFs on {node[u'host']}" + ) + + @staticmethod + def crypto_device_init(node, crypto_type, numvfs): + """Init Crypto QAT device virtual functions on DUT. + + :param node: DUT node. + :crypto_type: Crypto device type - HW_DH895xcc, HW_C3xxx, HW_C4xxx + or HW_4xxx. + :param numvfs: Number of VFs to initialize, 0 - disable the VFs. + :type node: dict + :type crypto_type: string + :type numvfs: int + :returns: nothing + :raises RuntimeError: If failed to stop VPP or QAT failed to initialize. + """ + if crypto_type == u"HW_DH895xcc": + kernel_mod = u"qat_dh895xcc" + kernel_drv = u"dh895xcc" + elif crypto_type == u"HW_C3xxx": + kernel_mod = u"qat_c3xxx" + kernel_drv = u"c3xxx" + elif crypto_type == u"HW_C4xxx": + kernel_mod = u"qat_c4xxx" + kernel_drv = u"c4xxx" + elif crypto_type == u"HW_4xxx": + kernel_mod = u"qat_4xxx" + kernel_drv = u"4xxx" + else: + raise RuntimeError( + f"Unsupported crypto device type on {node[u'host']}" + ) + + pci_addr = Topology.get_cryptodev(node) + + # QAT device must be re-bound to kernel driver before initialization. + DUTSetup.verify_kernel_module(node, kernel_mod, force_load=True) + + # Stop VPP to prevent deadlock. + VPPUtil.stop_vpp_service(node) + + current_driver = DUTSetup.get_pci_dev_driver( + node, pci_addr.replace(u":", r"\:") + ) + if current_driver is not None: + DUTSetup.pci_driver_unbind(node, pci_addr) + + # Bind to kernel driver. + DUTSetup.pci_driver_bind(node, pci_addr, kernel_drv) + + # Initialize QAT VFs. + if numvfs > 0: + DUTSetup.set_sriov_numvfs(node, pci_addr, numvfs) \ No newline at end of file diff --git a/resources/libraries/robot/shared/default.robot b/resources/libraries/robot/shared/default.robot index 44ade6dc33..755bc1a63d 100644 --- a/resources/libraries/robot/shared/default.robot +++ b/resources/libraries/robot/shared/default.robot @@ -35,6 +35,7 @@ | Library | resources.libraries.python.NodePath | Library | resources.libraries.python.Namespaces | Library | resources.libraries.python.PapiHistory +| Library | resources.libraries.python.QATUtil | Library | resources.libraries.python.SchedUtils | Library | resources.libraries.python.Tap | Library | resources.libraries.python.Tap.TapFeatureMask diff --git a/resources/libraries/robot/shared/suite_setup.robot b/resources/libraries/robot/shared/suite_setup.robot index 66efa34e69..9c8603af3b 100644 --- a/resources/libraries/robot/shared/suite_setup.robot +++ b/resources/libraries/robot/shared/suite_setup.robot @@ -282,7 +282,6 @@ | | ... | '${crypto_type}' == 'HW_4xxx' | ${16} | | Configure crypto device on all DUTs | ${crypto_type} | numvfs=${numvfs} | | ... | force_init=${True} -| | Configure kernel module on all DUTs | vfio_pci | force_load=${True} | Additional Suite Setup Action For nginx | | [Documentation] -- 2.16.6