From: juraj.linkes Date: Thu, 31 Jan 2019 14:32:24 +0000 (+0100) Subject: CSIT-1477: add 1n_tx2 VPP Device X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=commitdiff_plain;h=46efda9bd3a32cfe9944cb421ae3e2a3978bf9fc CSIT-1477: add 1n_tx2 VPP Device - add 1n_tx2 testbed - update VF reservation to meet 1n_tx2 testbed needs - update ansible with 1n_tx2 Change-Id: Ia075a913d4859f537fd0e6bff731ea88aff01dd9 Signed-off-by: juraj.linkes --- diff --git a/resources/libraries/bash/function/common.sh b/resources/libraries/bash/function/common.sh index 078ed70197..f1f02850e9 100644 --- a/resources/libraries/bash/function/common.sh +++ b/resources/libraries/bash/function/common.sh @@ -46,7 +46,7 @@ function activate_docker_topology () { device_image="$(< ${CSIT_DIR}/${IMAGE_VER_FILE})" case_text="${NODENESS}_${FLAVOR}" case "${case_text}" in - "1n_skx") + "1n_skx" | "1n_tx2") # We execute reservation over csit-shim-dcr (ssh) which runs sourced # script's functions. Env variables are read from ssh output # back to localhost for further processing. @@ -319,7 +319,7 @@ function deactivate_docker_topology () { case_text="${NODENESS}_${FLAVOR}" case "${case_text}" in - "1n_skx") + "1n_skx" | "1n_tx2") hostname=$(grep search /etc/resolv.conf | cut -d' ' -f3) || die ssh="ssh root@${hostname} -p 6022" env_vars=$(env | grep CSIT_ | tr '\n' ' ' ) || die @@ -436,6 +436,10 @@ function get_test_code () { NODENESS="1n" FLAVOR="skx" ;; + *"1n-tx2"*) + NODENESS="1n" + FLAVOR="tx2" + ;; *"2n-skx"*) NODENESS="2n" FLAVOR="skx" @@ -824,7 +828,7 @@ function select_vpp_device_tags () { TAGS=() - # We will prefix with perftest to prevent running other tests + # We will prefix with devicetest to prevent running other tests # (e.g. Functional). prefix="devicetestAND" if [[ "${TEST_CODE}" == "vpp-"* ]]; then @@ -895,7 +899,7 @@ function select_topology () { TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*vpp_device*.template ) TOPOLOGIES_TAGS="2_node_single_link_topo" ;; - "1n_skx") + "1n_skx" | "1n_tx2") TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*vpp_device*.template ) TOPOLOGIES_TAGS="2_node_single_link_topo" ;; diff --git a/resources/libraries/bash/function/device.sh b/resources/libraries/bash/function/device.sh index 0eda008abb..5d33af342e 100644 --- a/resources/libraries/bash/function/device.sh +++ b/resources/libraries/bash/function/device.sh @@ -270,7 +270,24 @@ function get_available_interfaces () { tg_netdev=(enp24) dut1_netdev=(enp59) ;; - "1n_vbox") + "1n_tx2") + # Add Intel Corporation XL710/X710 Virtual Function to the + # whitelist. + pci_id="0x154c" + tg_netdev=(enp5s2 enp5s3 enp5s4 enp5s5 + enp5s6 enp5s7 enp5s8 enp5s9) + tg_netdev+=(enp8s2 enp8s3 enp8s4 enp8s5 + enp8s6 enp8s7 enp8s8 enp8s9) + tg_netdev+=(enp8s10 enp8s11 enp8s12 enp8s13 + enp8s14 enp8s15 enp8s16 enp8s17) + dut1_netdev=(enp133s2 enp133s3 enp133s4 enp133s5 + enp133s6 enp133s7 enp133s8 enp133s9) + dut1_netdev+=(enp133s10 enp133s11 enp133s12 enp133s13 + enp133s14 enp133s15 enp133s16 enp133s17) + dut1_netdev+=(enp5s10 enp5s11 enp5s12 enp5s13 + enp5s14 enp5s15 enp5s16 enp5s17) + ;; + "1n_vbox") # Add Intel Corporation 82545EM Gigabit Ethernet Controller to the # whitelist. pci_id="0x100f" @@ -281,7 +298,7 @@ function get_available_interfaces () { die "Unknown specification: ${case_text}!" esac - net_path="/sys/bus/pci/devices/*/net/*" + device_count=2 # TG side of connections. TG_NETDEVS=() @@ -294,47 +311,40 @@ function get_available_interfaces () { DUT1_NETMACS=() DUT1_DRIVERS=() - # Following code is filtering available VFs represented by network device - # name. Only allowed VFs PCI IDs are used. - for netdev in \ - $(find ${net_path} -type d -name . -o -prune -exec basename '{}' ';'); + # Find the first ${device_count} number of available TG Linux network + # VF device names. Only allowed VF PCI IDs are filtered. + for netdev in ${tg_netdev[@]} do - if grep -q "${pci_id}" "/sys/class/net/${netdev}/device/device"; then - # We will filter to TG/DUT1 side of connection (this can be in - # future overriden by more advanced conditions for mapping). - for sub in ${tg_netdev[@]}; do - if [[ "${netdev#*$sub}" != "${netdev}" ]]; then - tg_side+=(${netdev}) - fi - done - for sub in ${dut1_netdev[@]}; do - if [[ "${netdev#*$sub}" != "${netdev}" ]]; then - dut1_side+=(${netdev}) - fi - done + for netdev_path in $(grep -l "${pci_id}" \ + /sys/class/net/${netdev}*/device/device \ + 2> /dev/null) + do + if [[ ${#TG_NETDEVS[@]} -lt ${device_count} ]]; then + tg_netdev_name=$(dirname ${netdev_path}) + tg_netdev_name=$(dirname ${tg_netdev_name}) + TG_NETDEVS+=($(basename ${tg_netdev_name})) + else + break + fi + done + if [[ ${#TG_NETDEVS[@]} -eq ${device_count} ]]; then + break fi done - case "${case_text}" in - "1n_skx") - # Pick up first two DUT1 interfaces binded to i40evf. - for netdev in "${dut1_side[@]::2}"; do - DUT1_NETDEVS+=(${netdev}) - done - # Corresponding TG interfaces will be same ID.SUB_ID, but on - # opposite linked device. - for netdev in "${DUT1_NETDEVS[@]}"; do - TG_NETDEVS+=(${netdev/$dut1_netdev/$tg_netdev}) - done - ;; - *) - for netdev in "${tg_side[@]::2}"; do - TG_NETDEVS+=(${netdev}) - done - for netdev in "${dut1_side[@]::2}"; do - DUT1_NETDEVS+=(${netdev}) - done - esac + i=0 + for netdev in "${TG_NETDEVS[@]}"; do + # Find the index of selected tg netdev among tg_netdevs + # e.g. enp8s5f7 is a vf of netdev enp8s5 with index 11 + # and the corresponding dut1 netdev is enp133s13. + while [[ "${netdev}" != "${tg_netdev[$i]}"* ]]; do + ((i++)) + done + # Rename tg netdev to dut1 netdev + # e.g. enp8s5f7 -> enp133s13f7 + DUT1_NETDEVS+=(${netdev/${tg_netdev[$i]}/${dut1_netdev[$i]}}) + # Don't need to reset i, all netdevs are sorted. + done for NETDEV in "${TG_NETDEVS[@]}"; do get_pci_addr diff --git a/resources/libraries/python/Constants.py b/resources/libraries/python/Constants.py index cb66a5d8d5..877fc25012 100644 --- a/resources/libraries/python/Constants.py +++ b/resources/libraries/python/Constants.py @@ -67,6 +67,9 @@ class Constants(object): # QEMU VM DPDK path QEMU_VM_DPDK = '/opt/dpdk-19.02' + # Docker container SUT image + DOCKER_SUT_IMAGE_UBUNTU = 'snergster/csit-sut:latest' + # TRex install version TREX_INSTALL_VERSION = '2.54' diff --git a/resources/libraries/python/ContainerUtils.py b/resources/libraries/python/ContainerUtils.py index 7d04b06fba..228648921a 100644 --- a/resources/libraries/python/ContainerUtils.py +++ b/resources/libraries/python/ContainerUtils.py @@ -784,7 +784,8 @@ class Docker(ContainerEngine): return if not self.container.image: - setattr(self.container, 'image', 'snergster/csit-sut:latest') + setattr(self.container, 'image', + Constants.DOCKER_SUT_IMAGE_UBUNTU) cmd = 'docker pull {image}'.format(image=self.container.image) diff --git a/resources/libraries/robot/shared/container.robot b/resources/libraries/robot/shared/container.robot index 6067a81c4e..1d646a6f22 100644 --- a/resources/libraries/robot/shared/container.robot +++ b/resources/libraries/robot/shared/container.robot @@ -56,12 +56,13 @@ | | | ${root}= | Run Keyword If | ${dut1_uuid_length} | | | ... | Get Docker Mergeddir | ${nodes['DUT1']} | ${dut1_uuid} | | | ... | ELSE | Set Variable | ${EMPTY} +| | | ${node_arch}= | Get Node Arch | ${nodes['${dut}']} | | | ${mnt}= | Create List | | | ... | ${root}/tmp/:/mnt/host/ | | | ... | ${root}/dev/vfio/:/dev/vfio/ | | | ... | ${root}/usr/bin/vpp:/usr/bin/vpp | | | ... | ${root}/usr/bin/vppctl:/usr/bin/vppctl -| | | ... | ${root}/usr/lib/x86_64-linux-gnu/:/usr/lib/x86_64-linux-gnu/ +| | | ... | ${root}/usr/lib/${node_arch}-linux-gnu/:/usr/lib/${node_arch}-linux-gnu/ | | | ... | ${root}/usr/share/vpp/:/usr/share/vpp/ | | | ${nf_cpus}= | Set Variable | ${None} | | | ${nf_cpus}= | Run Keyword If | ${pinning} diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.69.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.69.yaml new file mode 100644 index 0000000000..7d917c5e8b --- /dev/null +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.69.yaml @@ -0,0 +1,7 @@ +--- +# file: host_vars/10.30.51.69.yaml + +hostname: "s27-t13-sut1" +vfs_data_file: "csit-initialize-vfs-tx2.sh" +grub: + nr_hugepages: 57344 diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/hosts b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/hosts index 09bfc98ed9..80655bab27 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/hosts +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/hosts @@ -51,6 +51,11 @@ all: 10.32.8.11: null #s29-t26-sut1 10.32.8.12: null #s30-t35-sut1 10.32.8.13: null #s31-t35-sut2 + thunderx2: + children: + vpp_device: + hosts: + 10.30.51.69: null #s27-t13-sut1 taishan: children: # tg: diff --git a/resources/tools/testbed-setup/ansible/roles/common/files/grub_aarch64 b/resources/tools/testbed-setup/ansible/roles/common/files/grub_aarch64 new file mode 100644 index 0000000000..38a2cc38e0 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/common/files/grub_aarch64 @@ -0,0 +1,34 @@ +# If you change this file, run 'update-grub' afterwards to update +# /boot/grub/grub.cfg. +# For full documentation of the options in this file, see: +# info -f grub -n 'Simple configuration' + +GRUB_DEFAULT=0 +GRUB_TIMEOUT=10 +GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian` +GRUB_CMDLINE_LINUX_DEFAULT="console=ttyAMA0,115200n8" +GRUB_CMDLINE_LINUX="" + +# Uncomment to enable BadRAM filtering, modify to suit your needs +# This works with Linux (no patch required) and with any kernel that obtains +# the memory map information from GRUB (GNU Mach, kernel of FreeBSD ...) +#GRUB_BADRAM="0x01234567,0xfefefefe,0x89abcdef,0xefefefef" + +# Uncomment to disable graphical terminal (grub-pc only) +#GRUB_TERMINAL=console +GRUB_TERMINAL=serial +GRUB_SERIAL_COMMAND="serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1" + +# The resolution used on graphical terminal +# note that you can use only modes which your graphic card supports via VBE +# you can see them in real GRUB with the command `vbeinfo' +#GRUB_GFXMODE=640x480 + +# Uncomment if you don't want GRUB to pass "root=UUID=xxx" parameter to Linux +#GRUB_DISABLE_LINUX_UUID=true + +# Uncomment to disable generation of recovery mode menu entries +#GRUB_DISABLE_RECOVERY="true" + +# Uncomment to get a beep at grub start +#GRUB_INIT_TUNE="480 440 1" diff --git a/resources/tools/testbed-setup/ansible/roles/common/files/grub b/resources/tools/testbed-setup/ansible/roles/common/files/grub_x86_64 similarity index 100% rename from resources/tools/testbed-setup/ansible/roles/common/files/grub rename to resources/tools/testbed-setup/ansible/roles/common/files/grub_x86_64 diff --git a/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml index 43e83105ce..96e3f83e1b 100644 --- a/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml +++ b/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml @@ -77,7 +77,7 @@ - name: Copy grub file template: - src: 'files/grub' + src: 'files/grub_{{ ansible_machine }}' dest: '/etc/default/grub' owner: 'root' group: 'root' diff --git a/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs-default.sh b/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs-default.sh new file mode 100644 index 0000000000..becc18f1a8 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs-default.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +# Copyright (c) 2019 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Add Intel Corporation Ethernet Controller 10G X550T to blacklist. +PCI_BLACKLIST=($(lspci -Dmmd ':1563:0200' | cut -f1 -d' ')) + +# Add Intel Corporation Ethernet Controller X710 for 10GbE SFP+ to whitelist. +PCI_WHITELIST=($(lspci -Dmmd ':1572:0200' | cut -f1 -d' ')) + +# See http://pci-ids.ucw.cz/v2.2/pci.ids for more info. diff --git a/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs-tx2.sh b/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs-tx2.sh new file mode 100644 index 0000000000..591c95a2be --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs-tx2.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# Copyright (c) 2019 PANTHEON.tech and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Add QLogic Corp. FastLinQ QL41000 Series 10/25/40/50GbE Controller to +# blacklist. +PCI_BLACKLIST=($(lspci -Dmmd ':8070:0200' | cut -f1 -d' ')) + +# Add Intel Corporation Ethernet Controller XL710 for 40GbE QSFP+ to whitelist. +PCI_WHITELIST=($(lspci -Dmmd ':1583:0200' | cut -f1 -d' ')) + +# See http://pci-ids.ucw.cz/v2.2/pci.ids for more info. + +declare -A PF_INDICES +PF_INDICES["0000:05:00.0"]=0 +PF_INDICES["0000:05:00.1"]=1 +PF_INDICES["0000:08:00.0"]=2 +PF_INDICES["0000:08:00.1"]=1 +PF_INDICES["0000:85:00.0"]=0 +PF_INDICES["0000:85:00.1"]=2 diff --git a/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs.sh b/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs.sh index 06b9923128..365497865e 100644 --- a/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs.sh +++ b/resources/tools/testbed-setup/ansible/roles/vpp_device/files/csit-initialize-vfs.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Copyright (c) 2018 Cisco and/or its affiliates. +# Copyright (c) 2019 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -17,18 +17,13 @@ set -euo pipefail -# Add Intel Corporation Ethernet Controller 10G X550T to blacklist. -# See http://pci-ids.ucw.cz/v2.2/pci.ids for more info. -pci_blacklist=($(lspci -Dmmd ':1563:0200' | cut -f1 -d' ')) - -# Add Intel Corporation Ethernet Controller X710 for 10GbE SFP+ to whitelist. -# See http://pci-ids.ucw.cz/v2.2/pci.ids for more info. -pci_whitelist=($(lspci -Dmmd ':1572:0200' | cut -f1 -d' ')) +SCRIPT_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))" +source "${SCRIPT_DIR}/csit-initialize-vfs-data.sh" # Initilize whitelisted NICs with maximum number of VFs. pci_idx=0 -for pci_addr in ${pci_whitelist[@]}; do - if ! [[ ${pci_blacklist[*]} =~ "${pci_addr}" ]]; then +for pci_addr in ${PCI_WHITELIST[@]}; do + if ! [[ ${PCI_BLACKLIST[*]} =~ "${pci_addr}" ]]; then pci_path="/sys/bus/pci/devices/${pci_addr}" # SR-IOV initialization case "${1:-start}" in @@ -46,7 +41,12 @@ for pci_addr in ${pci_whitelist[@]}; do pf=$(basename "${pci_path}"/net/*) for vf in $(seq "${sriov_totalvfs}"); do # PCI address index in array (pairing siblings). - vlan_pf_idx=$(( pci_idx % (${#pci_whitelist[@]} / 2) )) + if [[ -n ${PF_INDICES[@]} ]] + then + vlan_pf_idx=${PF_INDICES[$pci_addr]} + else + vlan_pf_idx=$(( pci_idx % (${#PCI_WHITELIST[@]} / 2) )) + fi # 802.1Q base offset. vlan_bs_off=1100 # 802.1Q PF PCI address offset. diff --git a/resources/tools/testbed-setup/ansible/roles/vpp_device/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/vpp_device/tasks/main.yaml index 93c5d36bc2..46ca1d746b 100644 --- a/resources/tools/testbed-setup/ansible/roles/vpp_device/tasks/main.yaml +++ b/resources/tools/testbed-setup/ansible/roles/vpp_device/tasks/main.yaml @@ -1,6 +1,13 @@ --- # file: roles/vpp_device/tasks/main.yaml +- name: Load vfio-pci by default + lineinfile: + path: '/etc/modules' + state: 'present' + line: 'vfio-pci' + tags: load-vfio-pci + - name: Copy csit-initialize-vfs.sh copy: src: 'files/csit-initialize-vfs.sh' @@ -10,6 +17,26 @@ mode: '744' tags: copy-vf-script +- name: Copy csit-initialize-vfs-data.sh + copy: + src: 'files/{{ vfs_data_file }}' + dest: '/usr/local/bin/csit-initialize-vfs-data.sh' + owner: 'root' + group: 'root' + mode: '744' + tags: copy-vf-data-script + when: vfs_data_file is defined + +- name: Copy default csit-initialize-vfs-data.sh + copy: + src: 'files/csit-initialize-vfs-default.sh' + dest: '/usr/local/bin/csit-initialize-vfs-data.sh' + owner: 'root' + group: 'root' + mode: '744' + tags: copy-vf-data-script + when: vfs_data_file is not defined + - name: Start csit-initialize-vfs.service copy: src: 'files/csit-initialize-vfs.service'