From: Tibor Frank Date: Mon, 7 Oct 2024 14:12:52 +0000 (+0000) Subject: feat(new suite generator): initial commit X-Git-Url: https://gerrit.fd.io/r/gitweb?a=commitdiff_plain;h=2ec9880e10fa81abd976b43418fbefa275235be3;p=csit.git feat(new suite generator): initial commit Change-Id: I800c7d81075096d92bd7f9c90ab08f73cf33a9cb Signed-off-by: Tibor Frank --- diff --git a/resources/job_specifications/jobs.yaml b/resources/job_specifications/jobs.yaml new file mode 100644 index 0000000000..b3ec8c7361 --- /dev/null +++ b/resources/job_specifications/jobs.yaml @@ -0,0 +1,337 @@ +# See the documentation in +# "resources/libraries/python/suite_generator/suite_generator.py" + +# TODO: UPDATE + +# Jenkins / github actions jobs: +jobs: + + csit-dpdk-perf-mrr-weekly-master-{node-arch}: + test-type: mrr + framesize: [64, ] + core: [1, 2, 4] + node-arch: + - 2n-emr: 2n-emr-dpdk-iterative + - 2n-grc: 2n-grc-dpdk-iterative + - 2n-icx: 2n-icx-dpdk-iterative + - 2n-spr: 2n-spr-dpdk-iterative + - 2n-zn2: 2n-zn2-dpdk-iterative + - 3n-alt: 3n-alt-dpdk-iterative + - 3n-emr: 3n-emr-dpdk-iterative + - 3n-icx: 3n-icx-dpdk-iterative + - 3n-icxd: 3n-icxd-dpdk-iterative + - 3n-snr: 3n-snr-dpdk-iterative + - 3na-spr: 3na-spr-dpdk-iterative + - 3nb-spr: 3nb-spr-dpdk-iterative + + csit-dpdk-perf-report-iterative-{stream}-{node-arch}: + stream: + - "2506" + test-type: ndrpdr + framesize: # If the item is str or int, use 'core', + # if it is a dict, use the value. + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + core: [1, 2, 4] + node-arch: + - 2n-emr: 2n-emr-dpdk-iterative + - 2n-grc: 2n-grc-dpdk-iterative + - 2n-icx: 2n-icx-dpdk-iterative + - 2n-spr: 2n-spr-dpdk-iterative + - 2n-zn2: 2n-zn2-dpdk-iterative + - 3n-alt: 3n-alt-dpdk-iterative + - 3n-emr: 3n-emr-dpdk-iterative + - 3n-icx: 3n-icx-dpdk-iterative + - 3n-icxd: 3n-icxd-dpdk-iterative + - 3n-snr: 3n-snr-dpdk-iterative + - 3na-spr: 3na-spr-dpdk-iterative + - 3nb-spr: 3nb-spr-dpdk-iterative + + csit-dpdk-perf-report-coverage-{stream}-{node-arch}: + stream: + - "2506" + test-type: ndrpdr + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + core: [1, 2, 4] + node-arch: + - 2n-emr: 2n-emr-dpdk-iterative + - 2n-grc: 2n-grc-dpdk-iterative + - 2n-icx: 2n-icx-dpdk-iterative + - 2n-spr: 2n-spr-dpdk-iterative + - 2n-zn2: 2n-zn2-dpdk-iterative + - 3n-alt: 3n-alt-dpdk-iterative + - 3n-emr: 3n-emr-dpdk-iterative + - 3n-icx: 3n-icx-dpdk-iterative + - 3n-icxd: 3n-icxd-dpdk-iterative + - 3n-snr: 3n-snr-dpdk-iterative + - 3na-spr: 3na-spr-dpdk-iterative + - 3nb-spr: 3nb-spr-dpdk-iterative + + csit-dpdk-perf-verify-{stream}-{node-arch}: + stream: + - master + - "2506" + test-type: mrr + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + core: [1, 2, 4] + node-arch: + - 2n-emr: 2n-emr-dpdk-iterative + - 2n-grc: 2n-grc-dpdk-iterative + - 2n-icx: 2n-icx-dpdk-iterative + - 2n-spr: 2n-spr-dpdk-iterative + - 2n-zn2: 2n-zn2-dpdk-iterative + - 3n-alt: 3n-alt-dpdk-iterative + - 3n-emr: 3n-emr-dpdk-iterative + - 3n-icx: 3n-icx-dpdk-iterative + - 3n-icxd: 3n-icxd-dpdk-iterative + - 3n-snr: 3n-snr-dpdk-iterative + - 3na-spr: 3na-spr-dpdk-iterative + - 3nb-spr: 3nb-spr-dpdk-iterative + + csit-trex-perf-ndrpdr-weekly-master-{node-arch}: + test-type: ndrpdr + framesize: [64, ] + core: ["-", ] + node-arch: + - 2n-icx: 2n-icx-trex-iterative + - 2n-spr: 2n-spr-trex-iterative + + csit-trex-perf-report-coverage-{stream}-{node-arch}: + stream: + - "2506" + test-type: ndrpdr + framesize: [64, 1518, 9000, "imix"] + core: ["-", ] + node-arch: + - 2n-icx: 2n-icx-trex-coverage + - 2n-spr: 2n-spr-trex-coverage + + csit-trex-perf-report-iterative-{stream}-{node-arch}: + stream: + - "2506" + test-type: mrr + framesize: [64, ] + core: ["-", ] + node-arch: + - 2n-icx: 2n-icx-trex-iterative + - 2n-spr: 2n-spr-trex-iterative + + csit-trex-perf-verify-{stream}-{node-arch}: + stream: + - master + - "2506" + test-type: mrr + framesize: [64, ] + core: ["-", ] + node-arch: + - 2n-icx: 2n-icx-trex-iterative + - 2n-spr: 2n-spr-trex-iterative + + csit-vpp-perf-hoststack-daily-master-{node-arch}: + test-type: hoststack + framesize: [0, 2048] + core: [1, 2] + node-arch: + - 2n-emr: 2n-emr-vpp-hoststack + - 2n-grc: 2n-grc-vpp-hoststack + - 2n-icx: 2n-icx-vpp-hoststack + - 2n-spr: 2n-spr-vpp-hoststack + - 3n-emr: 3n-emr-vpp-hoststack + - 3n-icx: 3n-icx-vpp-hoststack + - 3na-spr: 3na-spr-vpp-hoststack + - 3nb-spr: 3nb-spr-vpp-hoststack + + csit-vpp-perf-mrr-daily-master-{node-arch}: + test-type: mrr + framesize: [64, ] + core: [1, 2, 4] + node-arch: + - 2n-emr: 2n-emr-vpp-iterative + - 2n-grc: 2n-grc-vpp-iterative + - 2n-icx: 2n-icx-vpp-iterative + - 2n-spr: 2n-spr-vpp-iterative + - 2n-zn2: 2n-zn2-vpp-iterative + - 3n-alt: 3n-alt-vpp-iterative + - 3n-emr: 3n-emr-vpp-iterative + - 3n-icx: 3n-icx-vpp-iterative + - 3n-icxd: 3n-icxd-vpp-iterative + - 3n-oct: 3n-oct-vpp-iterative + - 3n-snr: 3n-snr-vpp-iterative + - 3na-spr: 3na-spr-vpp-iterative + - 3nb-spr: 3nb-spr-vpp-iterative + + csit-vpp-perf-mrr-weekly-master-{node-arch}: + test-type: mrr + framesize: [64, ] + core: [1, 2, 4] + node-arch: + - 2n-aws: 2n-aws-vpp-iterative + - 2n-c6in: 2n-c6in-vpp-iterative + - 2n-c7gn: 2n-c7gn-vpp-iterative + - 2n-icx: 2n-icx-nfv + + csit-vpp-perf-ndrpdr-weekly-master-{node-arch}: + test-type: ndrpdr + framesize: [64, ] + core: [1, 2, 4] + node-arch: + - 2n-emr: 2n-emr-vpp-iterative + - 2n-grc: 2n-grc-vpp-iterative + - 2n-icx: 2n-icx-vpp-iterative + - 2n-spr: 2n-spr-vpp-iterative + - 2n-zn2: 2n-zn2-vpp-iterative + - 3n-alt: 3n-alt-vpp-iterative + - 3n-emr: 3n-emr-vpp-iterative + - 3n-icx: 3n-icx-vpp-iterative + - 3n-icxd: 3n-icxd-vpp-iterative + - 3n-oct: 3n-oct-vpp-iterative + - 3n-snr: 3n-snr-vpp-iterative + - 3na-spr: 3na-spr-vpp-iterative + - 3nb-spr: 3nb-spr-vpp-iterative + + csit-vpp-perf-report-coverage-{stream}-{node-arch}: + stream: + - "2506" + test-type: ndrpdr + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + core: [1, 2, 4] + node-arch: + - 2n-aws + - 2n-c6in + - 2n-c7gn + - 2n-emr + - 2n-grc + - 2n-icx + - 2n-spr + - 2n-zn2 + - 3n-alt + - 3n-emr + - 3n-icx + - 3n-icxd + - 3n-oct + - 3n-snr + - 3na-spr + - 3nb-spr + + csit-vpp-perf-report-iterative-{stream}-{node-arch}: + stream: + - "2506" + test-type: mrr + framesize: [64, ] + core: [1, 2, 4] + node-arch: + - 2n-aws: 2n-aws-vpp-iterative + - 2n-c6in: 2n-c6in-vpp-iterative + - 2n-c7gn: 2n-c7gn-vpp-iterative + - 2n-emr: 2n-emr-vpp-iterative + - 2n-grc: 2n-grc-vpp-iterative + - 2n-icx: 2n-icx-vpp-iterative + - 2n-spr: 2n-spr-vpp-iterative + - 2n-zn2: 2n-zn2-vpp-iterative + - 3n-alt: 3n-alt-vpp-iterative + - 3n-emr: 3n-emr-vpp-iterative + - 3n-icx: 3n-icx-vpp-iterative + - 3n-icxd: 3n-icxd-vpp-iterative + - 3n-oct: 3n-oct-vpp-iterative + - 3n-snr: 3n-snr-vpp-iterative + - 3na-spr: 3na-spr-vpp-iterative + - 3nb-spr: 3nb-spr-vpp-iterative + + csit-vpp-perf-soak-weekly-master-{node-arch}: + test-type: soak + framesize: [64, ] + core: [1, ] + node-arch: + - 2n-icx: 2n-icx-vpp-soak + - 2n-spr: 2n-spr-vpp-soak + - 3n-icx: 3n-icx-vpp-soak + - 3nd-icx: 3n-icxd-vpp-soak + + csit-vpp-perf-verify-{stream}-{node-arch}: + stream: + - master + - "2506" + test-type: mrr + framesize: [64, ] + core: [1, 2, 4] + node-arch: + - 2n-aws: 2n-aws-vpp-iterative + - 2n-c6in: 2n-c6in-vpp-iterative + - 2n-c7gn: 2n-c7gn-vpp-iterative + - 2n-emr: 2n-emr-vpp-iterative + - 2n-grc: 2n-grc-vpp-iterative + - 2n-icx: 2n-icx-vpp-iterative + - 2n-spr: 2n-spr-vpp-iterative + - 2n-zn2: 2n-zn2-vpp-iterative + - 3n-alt: 3n-alt-vpp-iterative + - 3n-emr: 3n-emr-vpp-iterative + - 3n-icx: 3n-icx-vpp-iterative + - 3n-icxd: 3n-icxd-vpp-iterative + - 3n-oct: 3n-oct-vpp-iterative + - 3n-snr: 3n-snr-vpp-iterative + - 3na-spr: 3na-spr-vpp-iterative + - 3nb-spr: 3nb-spr-vpp-iterative + + vpp-csit-bisect-master-ubuntu2404-aarch64-{node-arch}: + test-type: mrr + framesize: [64, ] + core: [1, 2, 4] + node-arch: + - 2n-grc: 2n-grc-vpp-iterative + - 3n-alt: 3n-alt-vpp-iterative + - 3n-oct: 3n-oct-vpp-iterative + + vpp-csit-bisect-master-ubuntu2404-x86_64-{node-arch}: + test-type: mrr + framesize: [64, ] + core: [1, 2, 4] + node-arch: + - 2n-emr: 2n-emr-vpp-iterative + - 2n-icx: 2n-icx-vpp-iterative + - 2n-spr: 2n-spr-vpp-iterative + - 2n-zn2: 2n-zn2-vpp-iterative + - 3n-emr: 3n-emr-vpp-iterative + - 3n-icx: 3n-icx-vpp-iterative + - 3n-icxd: 3n-icxd-vpp-iterative + - 3n-snr: 3n-snr-vpp-iterative + - 3na-spr: 3na-spr-vpp-iterative + - 3nb-spr: 3nb-spr-vpp-iterative + + vpp-csit-verify-perf-master-ubuntu2404-aarch64-{node-arch}: + test-type: mrr + framesize: [64, ] + core: [1, 2, 4] + node-arch: + - 2n-grc: 2n-grc-vpp-iterative + - 3n-alt: 3n-alt-vpp-iterative + - 3n-oct: 3n-oct-vpp-iterative + + vpp-csit-verify-perf-master-ubuntu2404-x86_64-{node-arch}: + test-type: mrr + framesize: [64, ] + core: [1, 2, 4] + node-arch: + - 2n-emr: 2n-emr-vpp-iterative + - 2n-icx: 2n-icx-vpp-iterative + - 2n-spr: 2n-spr-vpp-iterative + - 2n-zn2: 2n-zn2-vpp-iterative + - 3n-emr: 3n-emr-vpp-iterative + - 3n-icx: 3n-icx-vpp-iterative + - 3n-icxd: 3n-icxd-vpp-iterative + - 3n-snr: 3n-snr-vpp-iterative + - 3na-spr: 3na-spr-vpp-iterative + - 3nb-spr: 3nb-spr-vpp-iterative diff --git a/resources/job_specifications/test_groups.yaml b/resources/job_specifications/test_groups.yaml new file mode 100644 index 0000000000..ddbb5bb639 --- /dev/null +++ b/resources/job_specifications/test_groups.yaml @@ -0,0 +1,569 @@ +# See the documentation in +# "resources/libraries/python/suite_generator/suite_generator.py" + +# Definitions of test groups. +test-groups: + +# DPDK + dpdk-sm: + - eth-l2xcbase-testpmd + - ethip4-ip4base-l3fwd + + +# TRex + trex-ip4-sm: + - ethip4-ip4base-tg + - ethip4-ip4scale20k-tg + trex-nat44-cps-md: + - ethip4tcp-ip4base-h1024-p63-s64512-cps-tg + - ethip4tcp-ip4base-h262144-p63-s16515072-cps-tg + - ethip4tcp-ip4base-h1024-p63-s64512-cps-tg + - ethip4tcp-ip4base-h262144-p63-s16515072-cps-tg + trex-nat44-tput-md: + - ethip4tcp-ip4base-h1024-p63-s64512-tput-tg + - ethip4tcp-ip4base-h262144-p63-s16515072-tput-tg + - ethip4tcp-ip4base-h1024-p63-s64512-tput-tg + - ethip4tcp-ip4base-h262144-p63-s16515072-tput-tg + trex-ip6-sm: + - ethip6-ip6base-tg + - ethip6-ip6scale20k-tg + trex-l2-sm: + - eth-l2bdscale1mmaclrn-tg + + +# VPP + +## Container memif SW + container-memif-l2bd-sw-sm: + - eth-l2bdbasemaclrn-eth-2memif-1dcr + container-memif-sw-sm: + - eth-l2xcbase-eth-2memif-1dcr + - ethip4-ip4base-eth-2memif-1dcr +## Container memif HW + container-memif-hw-sm: + - eth-l2bdbasemaclrn-eth-2memif-dma-1dcr +## IPSec SW + ipsec-sw-sm: + - ethip4ipsec40tnlsw-ip4base-int-aes256gcm + ipsec-sw-md: + - ethip4ipsec4tnlsw-ip4base-int-aes256gcm + - ethip4ipsec40tnlsw-ip4base-int-aes256gcm + - ethip4ipsec10000tnlsw-ip4base-int-aes256gcm + # - ethip4ipsec10000tnlsw-ip4base-int-aes128cbc-hmac512sha + ipsec-sw-lg: + - ethip4ipsec4tnlsw-ip4base-int-aes128cbc-hmac512sha + - ethip4ipsec4tnlsw-ip4base-int-aes256gcm + - ethip4ipsec40tnlsw-ip4base-int-aes128cbc-hmac512sha + - ethip4ipsec40tnlsw-ip4base-int-aes256gcm + - ethip4ipsec1000tnlsw-ip4base-int-aes128cbc-hmac512sha + - ethip4ipsec1000tnlsw-ip4base-int-aes256gcm + - ethip4ipsec10000tnlsw-ip4base-int-aes128cbc-hmac512sha + - ethip4ipsec10000tnlsw-ip4base-int-aes256gcm + ipsec-sw-udir-xl: + - ethip4ipsec4tnlsw-ip4base-int-aes128cbc-hmac512sha-udir + - ethip4ipsec4tnlsw-ip4base-int-aes128gcm-udir + - ethip4ipsec4tnlsw-ip4base-int-aes256gcm-udir + - ethip4ipsec1000tnlsw-ip4base-int-aes128cbc-hmac512sha-udir + - ethip4ipsec1000tnlsw-ip4base-int-aes128gcm-udir + - ethip4ipsec1000tnlsw-ip4base-int-aes256gcm-udir + - ethip4ipsec10000tnlsw-ip4base-int-aes128cbc-hmac512sha-udir + - ethip4ipsec10000tnlsw-ip4base-int-aes128gcm-udir + - ethip4ipsec10000tnlsw-ip4base-int-aes256gcm-udir + ipsec-sw-policy-md: + - ethip4ipsec4tnlsw-ip4base-policy-aes256gcm + - ethip4ipsec10000tnlsw-ip4base-policy-aes256gcm + - ethip4ipsec10000tnlsw-ip4base-policy-flow-cache-aes256gcm + ipsec-sw-policy-xl: + - ethip4ipsec1tnlsw-ip4base-policy-aes256gcm-udir + - ethip4ipsec40tnlsw-ip4base-policy-aes256gcm-udir + - ethip4ipsec1000tnlsw-ip4base-policy-aes256gcm-udir + - ethip4ipsec1spe-ip4base-policy-outbound-nocrypto + - ethip4ipsec100spe-ip4base-policy-outbound-nocrypto + - ethip4ipsec1000spe-ip4base-policy-outbound-nocrypto + - ethip4ipsec1spe-cache-ip4base-policy-outbound-nocrypto + - ethip4ipsec100spe-cache-ip4base-policy-outbound-nocrypto + - ethip4ipsec1000spe-cache-ip4base-policy-outbound-nocrypto + ipsec-sw-async-sm: + - ethip4ipsec1tnlswasync-scheduler-ip4base-int-aes256gcm + - ethip4ipsec8tnlswasync-scheduler-ip4base-int-aes256gcm + ipsec-sw-async-2-sm: + - ethip4ipsec1tnlswasync-scheduler-ip4base-int-aes256gcm + - ethip4ipsec1tnlswasync-scheduler-ip4base-int-aes128cbc-hmac512sha + ipsec-sw-async-md: + - ethip4ipsec1tnlswasync-scheduler-ip4base-int-aes128cbc-hmac512sha + - ethip4ipsec8tnlswasync-scheduler-ip4base-int-aes128cbc-hmac512sha + - ethip4ipsec1tnlswasync-scheduler-ip4base-int-aes256gcm + - ethip4ipsec8tnlswasync-scheduler-ip4base-int-aes256gcm + ipsec-sw-fixtnlip-md: + - ethip4ipsec1000tnlsw-fixtnlip-ip4base-policy-aes256gcm + - ethip4ipsec1000tnlsw-fixtnlip-ip4base-policy-flow-dir-aes256gcm + - ethip4ipsec1000tnlsw-fixtnlip-ip4base-policy-flow-rss-aes256gcm + ipsec-sw-udp: + - ethip4ipsec10000tnlsw-udp-ip4base-int-aes256gcm + - ethip4ipsec10000tnlsw-udp-ar-ip4base-int-aes128cbc-hmac256sha + - ethip4ipsec10000tnlsw-udp-ar-ip4base-int-aes128cbc-hmac96sha + - ethip4ipsec10000tnlsw-udp-ar-ip4base-int-aes128ctr-hmac256sha + - ethip4ipsec10000tnlsw-udp-ar-ip4base-int-aes128ctr-hmac96sha + - ethip4ipsec10000tnlsw-udp-ar-ip4base-int-aes128gcm + - ethip4ipsec10000tnlsw-udp-ar-ip4base-int-aes128nullgmac + - ethip4ipsec10000tnlsw-udp-ar-ip4base-int-aes256cbc-hmac256sha + - ethip4ipsec10000tnlsw-udp-ar-ip4base-int-aes256cbc-hmac96sha + - ethip4ipsec10000tnlsw-udp-ar-ip4base-int-aes256gcm + - ethip4ipsec10000tnlsw-udp-ar-ip4base-int-aes256nullgmac + - ethip4ipsec10000tnlsw-udp-ar-ip4base-int-none-hmac96sha + ipsec-sw-reassembly-sm: + - ethip4ipsec1000tnlsw-ip4base-int-aes256gcm-reassembly + ipsec-sw-aes128cbc-hmac512sha-md: + - ethip4ipsec1tnlsw-ip4base-int-aes128cbc-hmac512sha + - ethip4ipsec4tnlsw-ip4base-int-aes128cbc-hmac512sha + - ethip4ipsec40tnlsw-ip4base-int-aes128cbc-hmac512sha + - ethip4ipsec1000tnlsw-ip4base-int-aes128cbc-hmac512sha + ipsec-sw-aes128cbc-hmac512sha-lg: + - ethip4ipsec1tnlsw-ip4base-int-aes128cbc-hmac512sha + - ethip4ipsec4tnlsw-ip4base-int-aes128cbc-hmac512sha + - ethip4ipsec40tnlsw-ip4base-int-aes128cbc-hmac512sha + - ethip4ipsec1000tnlsw-ip4base-int-aes128cbc-hmac512sha + - ethip4ipsec10000tnlsw-ip4base-int-aes128cbc-hmac512sha + - ethip4ipsec40000tnlsw-ip4base-int-aes128cbc-hmac512sha + ipsec-sw-aes128gcm-md: + - ethip4ipsec1tnlsw-ip4base-int-aes128gcm + - ethip4ipsec4tnlsw-ip4base-int-aes128gcm + - ethip4ipsec40tnlsw-ip4base-int-aes128gcm + - ethip4ipsec1000tnlsw-ip4base-int-aes128gcm + ipsec-sw-aes128gcm-lg: + - ethip4ipsec1tnlsw-ip4base-int-aes128gcm + - ethip4ipsec4tnlsw-ip4base-int-aes128gcm + - ethip4ipsec40tnlsw-ip4base-int-aes128gcm + - ethip4ipsec1000tnlsw-ip4base-int-aes128gcm + - ethip4ipsec10000tnlsw-ip4base-int-aes128gcm + - ethip4ipsec40000tnlsw-ip4base-int-aes128gcm + ipsec-sw-aes256gcm-md: + - ethip4ipsec1tnlsw-ip4base-int-aes256gcm + - ethip4ipsec4tnlsw-ip4base-int-aes256gcm + - ethip4ipsec40tnlsw-ip4base-int-aes256gcm + - ethip4ipsec1000tnlsw-ip4base-int-aes256gcm + ipsec-sw-aes256gcm-lg: + - ethip4ipsec1tnlsw-ip4base-int-aes256gcm + - ethip4ipsec4tnlsw-ip4base-int-aes256gcm + - ethip4ipsec40tnlsw-ip4base-int-aes256gcm + - ethip4ipsec1000tnlsw-ip4base-int-aes256gcm + - ethip4ipsec10000tnlsw-ip4base-int-aes256gcm + - ethip4ipsec40000tnlsw-ip4base-int-aes256gcm + ipsec-sw-policy-aes256gcm-sm: + - ethip4ipsec1tnlsw-ip4base-policy-aes256gcm + - ethip4ipsec1000tnlsw-ip4base-policy-aes256gcm + ipsec-sw-async-aes128cbc-hmac512sha-md: + - ethip4ipsec1tnlswasync-scheduler-ip4base-int-aes128cbc-hmac512sha + - ethip4ipsec2tnlswasync-scheduler-ip4base-int-aes128cbc-hmac512sha + - ethip4ipsec4tnlswasync-scheduler-ip4base-int-aes128cbc-hmac512sha + - ethip4ipsec8tnlswasync-scheduler-ip4base-int-aes128cbc-hmac512sha + ipsec-sw-async-aes128gcm-md: + - ethip4ipsec1tnlswasync-scheduler-ip4base-int-aes128gcm + - ethip4ipsec2tnlswasync-scheduler-ip4base-int-aes128gcm + - ethip4ipsec4tnlswasync-scheduler-ip4base-int-aes128gcm + - ethip4ipsec8tnlswasync-scheduler-ip4base-int-aes128gcm + ipsec-sw-async-aes256gcm-md: + - ethip4ipsec1tnlswasync-scheduler-ip4base-int-aes256gcm + - ethip4ipsec2tnlswasync-scheduler-ip4base-int-aes256gcm + - ethip4ipsec4tnlswasync-scheduler-ip4base-int-aes256gcm + - ethip4ipsec8tnlswasync-scheduler-ip4base-int-aes256gcm +## IPSec HW + ipsec-hw-sm: + - ethip4ipsec4tnlhwasync-ip4base-int-aes256gcm + - ethip4ipsec10000tnlhwasync-ip4base-int-aes256gcm + ipsec-hw-md: + - ethip4ipsec4tnlhwasync-ip4base-int-aes256gcm + - ethip4ipsec4tnlhwasync-ip4base-policy-aes256gcm + - ethip4ipsec10000tnlhwasync-ip4base-int-aes256gcm + - ethip4ipsec10000tnlhwasync-ip4base-policy-fastpath-aes256gcm + ipsec-hw-async-aes256gcm-md: + - ethip4ipsec1tnlhwasync-ip4base-int-aes256gcm + - ethip4ipsec4tnlhwasync-ip4base-int-aes256gcm + - ethip4ipsec1000tnlhwasync-ip4base-int-aes256gcm + - ethip4ipsec10000tnlhwasync-ip4base-int-aes256gcm + ipsec-hw-async-policy-aes256gcm-md: + - ethip4ipsec1tnlhwasync-ip4base-policy-aes256gcm + - ethip4ipsec4tnlhwasync-ip4base-policy-aes256gcm + - ethip4ipsec1000tnlhwasync-ip4base-policy-aes256gcm + - ethip4ipsec10000tnlhwasync-ip4base-policy-fastpath-aes256gcm +## GSO + gso-tap-sm: + - ethip4-ip4base-2tap-gso-iperf3 + - ethip4-ip4base-2tap-iperf3 + gso-vhost-sm: + - ethip4-ip4base-2vhost-gso-iperf3 + - ethip4-ip4base-2vhost-iperf3 +## Hoststack + hoststack-nginx-sm: + - eth-ip4tcphttp-ldpreload-nginx-1_21_5-cps + - eth-ip4tcphttp-ldpreload-nginx-1_21_5-rps + hoststack-nginx-md: + - eth-ip4tcphttp-ldpreload-nginx-1_21_5-cps + - eth-ip4tcphttp-ldpreload-nginx-1_21_5-rps + - eth-ip4tcphttp-dma-ldpreload-nginx-1_21_5-cps + - eth-ip4tcphttp-dma-ldpreload-nginx-1_21_5-rps + hoststack-iperf-md: + - eth-ip4tcpbase-ldpreload-iperf3 + - eth-ip4tcpscale1cl10s-ldpreload-iperf3 + - eth-ip4udpbase-ldpreload-iperf3 + - eth-ip4udpscale1cl10s-ldpreload-iperf3 + hoststack-vppecho-md: + - eth-ip4udpquicbase-vppecho + - eth-ip4udpquicscale1cl10s-vppecho + - eth-ip4udpquicscale10cl1s-vppecho + - eth-ip4udpquicscale10cl10s-vppecho +## IPv4 + ip4-base-sm: + - ethip4-ip4base + ip4-sm: + - ethip4-ip4base + - ethip4-ip4scale200k-rnd + ip4-aws-md: + - ethip4-ip4base + - ethip4-ip4scale20k + - ethip4-ip4scale20k-rnd + ip4-md: + - ethip4-ip4base + - ethip4-ip4scale20k-rnd + - ethip4-ip4scale2m-rnd + ip4-2-md: + - ethip4-ip4base + - ethip4-ip4scale20k-rnd + - ethip4-ip4scale200k-rnd + - ethip4-ip4scale2m-rnd + ip4-3-md: + - ethip4-ip4base + - ethip4-ip4scale2m + - ethip4-ip4scale2m-rnd + ip4-lg: + - ethip4-ip4base + - ethip4-ip4scale20k + - ethip4-ip4scale20k-rnd + - ethip4-ip4scale200k + - ethip4-ip4scale200k-rnd + - ethip4-ip4scale2m + - ethip4-ip4scale2m-rnd + ip4-60k-sm: + - ethip4-ip4scale60k-rnd +## IPv4 ACL + ip4-acl-sm: + - ethip4udp-ip4base-iacl50sf-10kflows + - ethip4udp-ip4base-iacl50sl-10kflows + ip4-acl-md: + - ethip4udp-ip4base-iacl50sf-10kflows + - ethip4udp-ip4base-iacl50sl-10kflows + - ethip4udp-ip4base-oacl50sf-10kflows + - ethip4udp-ip4base-oacl50sl-10kflows + ip4-acl-2-md: + - ethip4udp-ip4base-iacl1sf-10kflows + - ethip4udp-ip4base-iacl1sl-10kflows + - ethip4udp-ip4base-oacl1sf-10kflows + - ethip4udp-ip4base-oacl1sl-10kflows + ip4-acl-lg: + - ethip4udp-ip4base-iacl1sf-10kflows + - ethip4udp-ip4base-iacl1sl-10kflows + - ethip4udp-ip4base-iacl50sf-10kflows + - ethip4udp-ip4base-iacl50sl-10kflows + - ethip4udp-ip4base-oacl50sf-10kflows + - ethip4udp-ip4base-oacl50sl-10kflows + ip4-pol-sm: + - ethip4-ip4base-iacldstbase + ip4-pol-md: + - ethip4-ip4base-adlalwlistbase + - ethip4-ip4base-iacldstbase + - ethip4-ip4base-ipolicemarkbase +## IPv4 NAT44 + nat44det-sm: + - ethip4udp-nat44det-h65536-p63-s4128758 + nat44ed-cps-sm: + - ethip4tcp-nat44ed-h65536-p63-s4128768-cps + - ethip4udp-nat44ed-h65536-p63-s4128768-cps + nat44ed-tput-sm: + - ethip4tcp-nat44ed-h65536-p63-s4128768-tput + - ethip4udp-nat44ed-h65536-p63-s4128768-tput + nat44-udir-sm: + - ethip4udp-nat44ed-h65536-p63-s4128768-udir + nat44-ip4base-cps-sm: + - ethip4tcp-ip4base-h65536-p63-s4128768-cps + - ethip4udp-ip4base-h65536-p63-s4128768-cps + nat44-ip4base-cps-md: + - ethip4tcp-ip4base-h65536-p63-s4128768-cps + - ethip4udp-ip4base-h1024-p63-s64512-cps + - ethip4udp-ip4base-h16384-p63-s1032192-cps + - ethip4udp-ip4base-h65536-p63-s4128768-cps + - ethip4udp-ip4base-h262144-p63-s16515072-cps + nat44-ip4base-tput-sm: + - ethip4tcp-ip4base-h65536-p63-s4128768-tput + - ethip4udp-ip4base-h65536-p63-s4128768-tput + nat44-ip4base-tput-md: + - ethip4tcp-ip4base-h65536-p63-s4128768-tput + - ethip4udp-ip4base-h1024-p63-s64512-tput + - ethip4udp-ip4base-h16384-p63-s1032192-tput + - ethip4udp-ip4base-h65536-p63-s4128768-tput + - ethip4udp-ip4base-h262144-p63-s16515072-tput +## IPv4 Tunnels - Lisp + lisp-sm: + - ethip4lispip4-ip4base + - ethip4lispip6-ip4base +## IPv4 Tunnels - Geneve + geneve-md: + - ethip4--ethip4udpgeneve-1tun-ip4base + - ethip4--ethip4udpgeneve-16tun-ip4base + - ethip4--ethip4udpgeneve-256tun-ip4base + geneve-lg: + - ethip4--ethip4udpgeneve-1tun-ip4base + - ethip4--ethip4udpgeneve-4tun-ip4base + - ethip4--ethip4udpgeneve-16tun-ip4base + - ethip4--ethip4udpgeneve-64tun-ip4base + - ethip4--ethip4udpgeneve-256tun-ip4base +## IPv4 Tunnels - VXLan + vxlan-sm: + - ethip4vxlan-l2bdbasemaclrn + - ethip4vxlan-l2xcbase +## IPv4 Tunnels GTPU SW + gtpu-sw-sm: + - ethip4gtpusw-ip4base + gtpu-sw-reassembly-sm: + - ethip4gtpusw-ip4base-reassembly +## IPv4 Tunnels GTPU HW + gtpu-hw-sm: + - ethip4gtpuhw-ip4base +## IPv4 Tunnels - Wireguard SW + wireguard-sw-sm: + - ethip4udpwireguard1tnlsw-ip4base + wireguard-sw-md: + - ethip4udpwireguard1tnlsw-ip4base + - ethip4udpwireguard4tnlsw-ip4base + - ethip4udpwireguard100tnlsw-ip4base + - ethip4udpwireguard1000tnlsw-ip4base + wireguard-sw-lg: + - ethip4udpwireguard1tnlsw-ip4base + - ethip4udpwireguard2tnlsw-ip4base + - ethip4udpwireguard4tnlsw-ip4base + - ethip4udpwireguard8tnlsw-ip4base + - ethip4udpwireguard100tnlsw-ip4base + - ethip4udpwireguard1000tnlsw-ip4base +## IPv4 Tunnels - Wireguard HW + wireguard-hw-md: + - ethip4udpwireguard1tnlhwasync-ip4base + - ethip4udpwireguard4tnlhwasync-ip4base + - ethip4udpwireguard100tnlhwasync-ip4base + - ethip4udpwireguard1000tnlhwasync-ip4base + wireguard-hw-lg: + - ethip4udpwireguard1tnlhwasync-ip4base + - ethip4udpwireguard2tnlhwasync-ip4base + - ethip4udpwireguard4tnlhwasync-ip4base + - ethip4udpwireguard8tnlhwasync-ip4base + - ethip4udpwireguard100tnlhwasync-ip4base + - ethip4udpwireguard1000tnlhwasync-ip4base +## IPv6 + ip6-base-sm: + - ethip6-ip6base + ip6-sm: + - ethip6-ip6base + - ethip6-ip6scale200k-rnd + ip6-aws-md: + - ethip6-ip6base + - ethip6-ip6scale20k + - ethip6-ip6scale20k-rnd + ip6-2m-sm: + - ethip6-ip6scale2m + - ethip6-ip6scale2m-rnd + ip6-md: + - ethip6-ip6base + - ethip6-ip6scale20k-rnd + - ethip6-ip6scale2m-rnd + ip6-acl-md: + - ethip6-ip6base + - ethip6-ip6base-adlalwlistbase + - ethip6-ip6base-iacldstbase + ip6-scale-md: + - ethip6-ip6scale20k + - ethip6-ip6scale200k + - ethip6-ip6scale2m + ip6-scale-lg: + - ethip6-ip6scale20k + - ethip6-ip6scale20k-rnd + - ethip6-ip6scale200k + - ethip6-ip6scale200k-rnd + - ethip6-ip6scale2m + - ethip6-ip6scale2m-rnd +## IPv6 Tunnels + ip6-lisp-sm: + - ethip6lispip4-ip6base + - ethip6lispip6-ip6base +## L2 + l2-base-sm: + - eth-l2patch + - eth-l2xcbase + - eth-l2bdbasemaclrn + l2-base-md: + - eth-l2patch + - eth-l2xcbase + - eth-l2bdbasemaclrn + - dot1q-l2bdbasemaclrn + l2-scale-md: + - eth-l2bdscale10kmaclrn + - eth-l2bdscale100kmaclrn + - eth-l2bdscale1mmaclrn + l2-sm: + - eth-l2xcbase + - eth-l2bdbasemaclrn + l2-md: + - eth-l2xcbase + - eth-l2bdbasemaclrn + - eth-l2bdscale10kmaclrn + - eth-l2bdscale100kmaclrn + l2-lg: + - eth-l2xcbase + - eth-l2patch + - eth-l2bdbasemaclrn + - dot1q-l2bdbasemaclrn + - eth-l2bdscale10kmaclrn + # - eth-l2bdscale100kmaclrn + - eth-l2bdscale1mmaclrn + l2-1lbvpplacp-sm: + - eth-l2xcbase-1lbvpplacp +## L2 ACL + l2-acl-md: + - eth-l2bdbasemaclrn-iacl50sf-10kflows + - eth-l2bdbasemaclrn-iacl50sl-10kflows + - eth-l2bdbasemaclrn-oacl50sf-10kflows + - eth-l2bdbasemaclrn-oacl50sl-10kflows + l2-acl-2-md: + - eth-l2bdbasemaclrn-iacl1sf-10kflows + - eth-l2bdbasemaclrn-iacl1sl-10kflows + - eth-l2bdbasemaclrn-oacl1sf-10kflows + - eth-l2bdbasemaclrn-oacl1sl-10kflows + l2-acl-xl: + - eth-l2bdbasemaclrn-iacl1sf-10kflows + - eth-l2bdbasemaclrn-iacl1sl-10kflows + - eth-l2bdbasemaclrn-iacl10sf-10kflows + - eth-l2bdbasemaclrn-iacl10sl-10kflows + - eth-l2bdbasemaclrn-iacl50sf-10kflows + - eth-l2bdbasemaclrn-iacl50sl-10kflows + - eth-l2bdbasemaclrn-oacl1sf-10kflows + - eth-l2bdbasemaclrn-oacl1sl-10kflows + - eth-l2bdbasemaclrn-oacl10sf-10kflows + - eth-l2bdbasemaclrn-oacl10sl-10kflows + - eth-l2bdbasemaclrn-oacl50sf-10kflows + - eth-l2bdbasemaclrn-oacl50sl-10kflows + l2-macip-sm: + - eth-l2bdbasemaclrn-macip-iacl1sl-10kflows + l2-macip-md: + - eth-l2bdbasemaclrn-macip-iacl1sl-10kflows + - eth-l2bdbasemaclrn-macip-iacl10sl-10kflows + - eth-l2bdbasemaclrn-macip-iacl50sl-10kflows +## LB + lb-md: + - ethip4-loadbalancer-l3dsr + - ethip4-loadbalancer-maglev + - ethip4-loadbalancer-nat4 +## SRv6 + srv6-lg: + - ethip6ip6-ip6base-srv6enc1sid + - ethip6srhip6-ip6base-srv6enc2sids + - ethip6srhip6-ip6base-srv6enc2sids-nodecaps + - ethip6srhip6-ip6base-srv6proxy-dyn + - ethip6srhip6-ip6base-srv6proxy-masq + - ethip6srhip6-ip6base-srv6proxy-stat +## VM Vhost + vhost-sm: + - eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm + - eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm-vppl2xc + vhost-md: + - eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm + - eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm-vppl2xc + - eth-l2xcbase-eth-2vhostvr1024-1vm + - eth-l2xcbase-eth-2vhostvr1024-1vm-vppl2xc + vhost-2-md: + - eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm-vppl2xc + - eth-l2xcbase-eth-2vhostvr1024-1vm-vppl2xc + - ethip4-ip4base-eth-2vhostvr1024-1vm-vppip4 + vhost-lg: + - eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm + - eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm-vppl2xc + - eth-l2xcbase-eth-2vhostvr1024-1vm + - eth-l2xcbase-eth-2vhostvr1024-1vm-vppl2xc + - ethip4-ip4base-eth-2vhostvr1024-1vm + - ethip4-ip4base-eth-2vhostvr1024-1vm-vppip4 + vhost-vxlan-sm: + - ethip4vxlan-l2bdbasemaclrn-eth-2vhostvr1024-1vm + - ethip4vxlan-l2bdbasemaclrn-eth-2vhostvr1024-1vm-vppl2xc + vhost-vxlan-2-sm: + - ethip4vxlan-l2bdbasemaclrn-eth-2vhostvr1024-1vm-vppl2xc +## soak + soak-memif-sm: + - eth-l2bdbasemaclrn-eth-2memif-1dcr + soak-memif-dma-sm: + - eth-l2bdbasemaclrn-eth-2memif-dma-1dcr + soak-crypto-sm: + - ethip4ipsec1000tnlsw-ip4base-int-aes128cbc-hmac512sha + - ethip4ipsec1000tnlsw-ip4base-int-aes256gcm + soak-crypto-md: + - ethip4ipsec1000tnlsw-ip4base-int-aes128cbc-hmac512sha + - ethip4ipsec1000tnlsw-ip4base-int-aes256gcm + - ethip4ipsec1000tnlhwasync-ip4base-int-aes128cbc-hmac512sha + - ethip4ipsec1000tnlhwasync-ip4base-int-aes256gcm + soak-ip4-sm: + - ethip4-ip4scale200k-rnd + soak-ip4-md: + - ethip4-ip4base + - ethip4-ip4scale20k-rnd + - ethip4-ip4scale200k-rnd + soak-nat44-cps-sm: + - ethip4tcp-nat44ed-h65536-p63-s4128768-cps + - ethip4udp-nat44ed-h65536-p63-s4128768-cps + soak-nat44-tput-sm: + - ethip4tcp-nat44ed-h65536-p63-s4128768-tput + - ethip4udp-nat44ed-h65536-p63-s4128768-tput + soak-ip6-sm: + - ethip6-ip6scale200k-rnd + soak-ip6-md: + - ethip6-ip6base + - ethip6-ip6scale20k-rnd + - ethip6-ip6scale200k-rnd + soak-l2-sm: + - eth-l2bdbasemaclrn + soak-l2-md: + - eth-l2xcbase + - eth-l2bdbasemaclrn + - eth-l2bdscale1mmaclrn + soak-srv6-sm: + - ethip6srhip6-ip6base-srv6enc2sids + - ethip6srhip6-ip6base-srv6proxy-masq + soak-vhost-sm: + - eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm-vppl2xc +## nfv density + nfv-dcr-memif-chain-lg: + - eth-l2bd-1ch-2mif-1dcr1t-vppip4 + - eth-l2bd-1ch-4mif-2dcr1t-vppip4 + - eth-l2bd-1ch-8mif-4dcr1t-vppip4 + - eth-l2bd-1ch-12mif-6dcr1t-vppip4 + - eth-l2bd-2ch-4mif-2dcr1t-vppip4 + - eth-l2bd-2ch-8mif-4dcr1t-vppip4 + - eth-l2bd-4ch-8mif-4dcr1t-vppip4 + - eth-l2bd-6ch-12mif-6dcr1t-vppip4 + nfv-dcr-memif-pipeline-lg: + - eth-l2bd-1pl-2mif-1dcr1t-vppip4 + - eth-l2bd-1pl-2mif-2dcr1t-vppip4 + - eth-l2bd-1pl-2mif-4dcr1t-vppip4 + - eth-l2bd-1pl-2mif-6dcr1t-vppip4 + - eth-l2bd-2pl-4mif-2dcr1t-vppip4 + - eth-l2bd-2pl-4mif-4dcr1t-vppip4 + - eth-l2bd-2pl-8mif-4dcr1t-vppip4 + - eth-l2bd-6pl-12mif-6dcr1t-vppip4 + nfv-vm-vhost-chain-lg: + - eth-l2bd-1ch-2vh-1vm1t-vppip4 + - eth-l2bd-1ch-4vh-2vm1t-vppip4 + - eth-l2bd-1ch-8vh-4vm1t-vppip4 + - eth-l2bd-1ch-12vh-6vm1t-vppip4 + - eth-l2bd-2ch-4vh-2vm1t-vppip4 + - eth-l2bd-2ch-8vh-4vm1t-vppip4 + - eth-l2bd-4ch-8vh-4vm1t-vppip4 + - eth-l2bd-6ch-12vh-6vm1t-vppip4 + nfv-vm-vhost-chain-vxlan-md: + - dot1qip4vxlan-l2bd-1ch-2vh-1vm1t-testpmd + - dot1qip4vxlan-l2bd-2ch-4vh-2vm1t-testpmd + - dot1qip4vxlan-l2bd-4ch-8vh-4vm1t-testpmd + - dot1qip4vxlan-l2bd-6ch-12vh-6vm1t-testpmd diff --git a/resources/job_specifications/test_sets.yaml b/resources/job_specifications/test_sets.yaml new file mode 100644 index 0000000000..3cd1f13a03 --- /dev/null +++ b/resources/job_specifications/test_sets.yaml @@ -0,0 +1,3955 @@ +# See the documentation in +# "resources/libraries/python/suite_generator/suite_generator.py" + +# Test groups assigned to infrastructure. +test-sets: + + 2n-emr-dpdk-iterative: + infra: + 100ge2p1e810cq: + - vfio-pci + tests: + - dpdk-sm + + 2n-grc-dpdk-iterative: + infra: + 200ge2p1cx7veat: + - mlx5_core + tests: + - dpdk-sm + + 2n-icx-dpdk-iterative: + infra: + 200ge2p1cx7veat: + - mlx5_core + 25ge2p1e810xxv: + - vfio-pci + 100ge2p1e810cq: + - vfio-pci + tests: + - dpdk-sm + + 2n-spr-dpdk-iterative: + infra: + 200ge2p1cx7veat: + - mlx5_core + 25ge2p1e810xxv: + - vfio-pci + 100ge2p1e810cq: + - vfio-pci + tests: + - dpdk-sm + + 2n-zn2-dpdk-iterative: + infra: + 25ge2p1xxv710: + - vfio-pci + tests: + - dpdk-sm + + 3n-alt-dpdk-iterative: + infra: + 100ge2p1cx6dx: + - mlx5_core + tests: + - dpdk-sm + + 3n-emr-dpdk-iterative: + infra: + 100ge2p1e810cq: + - vfio-pci + tests: + - dpdk-sm + + 3n-icx-dpdk-iterative: + infra: + 100ge2p1cx6dx: + - mlx5_core + 25ge2p1e810xxv: + - vfio-pci + 100ge2p1e810cq: + - vfio-pci + tests: + - dpdk-sm + + 3n-icxd-dpdk-iterative: + infra: + 25ge2p1e823c: + - vfio-pci + tests: + - dpdk-sm + + 3n-snr-dpdk-iterative: + infra: + 25ge2p1e822cq: + - vfio-pci + tests: + - dpdk-sm + + 3na-spr-dpdk-iterative: + infra: + 200ge2p1cx7veat: + - mlx5_core + tests: + - dpdk-sm + + 3nb-spr-dpdk-iterative: + infra: + 25ge2p1e822cq: + - vfio-pci + tests: + - dpdk-sm + + + # Test sets for dpdk-coverage - the same as iterative + + + 2n-icx-trex-iterative: + infra: + 100ge2p1e810cq: + - "-" + tests: + - trex-ip4-sm + - trex-nat44-cps-md + - trex-nat44-tput-md: + framesize: [100, ] + - trex-ip6-sm: + framesize: [78, ] + - trex-l2-sm + + 2n-spr-trex-iterative: + infra: + 100ge2p1e810cq: + - "-" + 200ge2p1cx7veat: + - "-" + tests: + - trex-ip4-sm + - trex-nat44-cps-md + - trex-nat44-tput-md: + framesize: [100, ] + - trex-ip6-sm: + framesize: [78, ] + - trex-l2-sm + + 2n-icx-trex-coverage: + infra: + 100ge2p1e810cq: + - "-" + tests: + - trex-ip4-sm + - trex-nat44-cps-md + - trex-nat44-tput-md: + framesize: [100, ] + - trex-ip6-sm: + framesize: [78, 1518, 9000, "imix"] + - trex-l2-sm + + 2n-spr-trex-coverage: + infra: + 100ge2p1e810cq: + - "-" + 200ge2p1cx7veat: + - "-" + tests: + - trex-ip4-sm + - trex-nat44-cps-md + - trex-nat44-tput-md: + framesize: [100, ] + - trex-ip6-sm: + framesize: [78, 1518, 9000, "imix"] + - trex-l2-sm + + + 2n-emr-vpp-hoststack: + framesize: [0, 2048] + core: [1, 2] + infra: + 100ge2p1e810cq: + - vfio-pci + tests: + - hoststack-nginx-md + + 2n-grc-vpp-hoststack: + framesize: [0, 2048] + core: [1, 2] + infra: + 200ge2p1cx7veat: + - mlx5_core + tests: + - hoststack-nginx-sm + + 2n-icx-vpp-hoststack: + framesize: [0, 2048] + core: [1, 2] + infra: + 25ge2p1e810xxv: + - vfio-pci + 100ge2p1e810cq: + - vfio-pci + 200ge2p1cx7veat: + - mlx5_core + tests: + - hoststack-nginx-sm + + 2n-spr-vpp-hoststack: + framesize: [0, 2048] + core: [1, 2] + infra: + 100ge2p1e810cq: + - vfio-pci + 200ge2p1cx7veat: + - mlx5_core + tests: + - hoststack-nginx-md + + 3n-emr-vpp-hoststack: + infra: + 100ge2p1e810cq: + - vfio-pci + core: [1, ] + tests: + - hoststack-iperf-md: + framesize: [1460, ] + - hoststack-vppecho-md: + framesize: [1280, ] + + 3n-icx-vpp-hoststack: + infra: + 100ge2p1e810cq: + - vfio-pci + 100ge2p1cx6dx: + - mlx5_core + core: [1, ] + tests: + - hoststack-iperf-md: + framesize: [1460, ] + - hoststack-vppecho-md: + framesize: [1280, ] + + 3na-spr-vpp-hoststack: + infra: + 200ge2p1cx7veat: + - mlx5_core + core: [1, ] + tests: + - hoststack-iperf-md: + framesize: [1460, ] + - hoststack-vppecho-md: + framesize: [1280, ] + + 3nb-spr-vpp-hoststack: + infra: + 100ge2p1e810cq: + - vfio-pci + core: [1, ] + tests: + - hoststack-iperf-md: + framesize: [1460, ] + - hoststack-vppecho-md: + framesize: [1280, ] + + + 2n-emr-vpp-cov-ip4-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-lg + + 2n-emr-vpp-cov-ip4-01: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-pol-md + + 2n-emr-vpp-cov-ip4-02: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-acl-lg + + 2n-emr-vpp-cov-ip4-03: + infra: + 100ge2p1e810cq: + - avf + framesize: [64, imix] + tests: + - nat44det-sm + - nat44-udir-sm + - nat44ed-cps-sm: + framesize: [64, ] + - nat44ed-tput-sm: + framesize: [100, ] + - nat44-ip4base-cps-sm: + framesize: [64, ] + - nat44-ip4base-tput-sm: + framesize: [100, ] + + 2n-emr-vpp-cov-ip4tun-00: + infra: + 100ge2p1e810cq: + - avf + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - geneve-lg + + 2n-emr-vpp-cov-ip6-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-acl-md + + 2n-emr-vpp-cov-ip6-01: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-scale-lg + + 2n-emr-vpp-cov-l2-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-base-md + + 2n-emr-vpp-cov-l2-01: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-scale-md + + 2n-emr-vpp-cov-lb-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - lb-md + + 2n-emr-vpp-cov-memif-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - container-memif-l2bd-sw-sm + - container-memif-sw-sm + - container-memif-hw-sm + + 2n-emr-vpp-cov-vhost-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - vhost-lg + + + 2n-grc-vpp-cov-ip4-00: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-lg + + 2n-grc-vpp-cov-ip4-01: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-pol-md + + 2n-grc-vpp-cov-ip4-02: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-acl-lg + + 2n-grc-vpp-cov-ip4-03: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: [64, imix] + tests: + - nat44det-sm + - nat44-udir-sm + - nat44ed-cps-sm: + framesize: [64, ] + - nat44ed-tput-sm: + framesize: [100, ] + - nat44-ip4base-cps-sm: + framesize: [64, ] + - nat44-ip4base-tput-sm: + framesize: [100, ] + + 2n-grc-vpp-cov-ip4tun-00: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - geneve-lg + + 2n-grc-vpp-cov-ip6-00: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-acl-md + + 2n-grc-vpp-cov-ip6-01: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-scale-lg + + 2n-grc-vpp-cov-l2-00: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-base-md + + 2n-grc-vpp-cov-l2-01: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-scale-md + + 2n-grc-vpp-cov-lb-00: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - lb-md + + 2n-grc-vpp-cov-memif-00: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - container-memif-l2bd-sw-sm + - container-memif-sw-sm + - container-memif-hw-sm + + 2n-grc-vpp-cov-vhost-00: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - vhost-lg + + + 2n-icx-vpp-cov-ip4-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-lg + + 2n-icx-vpp-cov-ip4-01: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-pol-md + + 2n-icx-vpp-cov-ip4-02: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-acl-lg + + 2n-icx-vpp-cov-ip4-03: + infra: + 100ge2p1e810cq: + - avf + framesize: [64, imix] + tests: + - nat44det-sm + - nat44-udir-sm + - nat44ed-cps-sm: + framesize: [64, ] + - nat44ed-tput-sm: + framesize: [100, ] + - nat44-ip4base-cps-sm: + framesize: [64, ] + - nat44-ip4base-tput-sm: + framesize: [100, ] + + 2n-icx-vpp-cov-ip4-10: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-lg + + 2n-icx-vpp-cov-ip4-11: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-pol-md + + 2n-icx-vpp-cov-ip4-12: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-acl-lg + + 2n-icx-vpp-cov-ip4tun-00: + infra: + 100ge2p1e810cq: + - avf + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - geneve-lg + + 2n-icx-vpp-cov-ip6-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-acl-md + + 2n-icx-vpp-cov-ip6-01: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-scale-lg + + 2n-icx-vpp-cov-ip6-10: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-acl-md + + 2n-icx-vpp-cov-ip6-11: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-scale-lg + + 2n-icx-vpp-cov-l2-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-base-md + + 2n-icx-vpp-cov-l2-01: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-scale-md + + 2n-icx-vpp-cov-l2-10: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-base-md + + 2n-icx-vpp-cov-l2-11: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-scale-md + + 2n-icx-vpp-cov-lb-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - lb-md + + 2n-icx-vpp-cov-lb-10: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - lb-md + + 2n-icx-vpp-cov-memif-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - container-memif-l2bd-sw-sm + - container-memif-sw-sm + + 2n-icx-vpp-cov-memif-10: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - container-memif-l2bd-sw-sm + - container-memif-sw-sm + + 2n-icx-vpp-cov-vhost-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - vhost-lg + + 2n-icx-vpp-cov-vhost-10: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - vhost-lg + + + 2n-spr-vpp-cov-ip4-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-lg + + 2n-spr-vpp-cov-ip4-01: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-pol-md + + 2n-spr-vpp-cov-ip4-02: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-acl-lg + + 2n-spr-vpp-cov-ip4-03: + infra: + 100ge2p1e810cq: + - avf + framesize: [64, imix] + tests: + - nat44det-sm + - nat44-udir-sm + - nat44ed-cps-sm: + framesize: [64, ] + - nat44ed-tput-sm: + framesize: [100, ] + - nat44-ip4base-cps-sm: + framesize: [64, ] + - nat44-ip4base-tput-sm: + framesize: [100, ] + + 2n-spr-vpp-cov-ip4-10: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-lg + + 2n-spr-vpp-cov-ip4-11: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-pol-md + + 2n-spr-vpp-cov-ip4-12: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-acl-lg + + 2n-spr-vpp-cov-ip4tun-00: + infra: + 100ge2p1e810cq: + - avf + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - geneve-lg + + 2n-spr-vpp-cov-ip6-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-acl-md + + 2n-spr-vpp-cov-ip6-01: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-scale-lg + + 2n-spr-vpp-cov-ip6-10: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-acl-md + + 2n-spr-vpp-cov-ip6-11: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-scale-lg + + 2n-spr-vpp-cov-l2-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-base-md + + 2n-spr-vpp-cov-l2-01: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-scale-md + + 2n-spr-vpp-cov-l2-10: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-base-md + + 2n-spr-vpp-cov-l2-11: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-scale-md + + 2n-spr-vpp-cov-lb-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - lb-md + + 2n-spr-vpp-cov-lb-10: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - lb-md + + 2n-spr-vpp-cov-memif-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - container-memif-l2bd-sw-sm + - container-memif-sw-sm + - container-memif-hw-sm + + 2n-spr-vpp-cov-memif-10: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - container-memif-l2bd-sw-sm + - container-memif-sw-sm + - container-memif-hw-sm + + 2n-spr-vpp-cov-vhost-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - vhost-lg + + 2n-spr-vpp-cov-vhost-10: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - vhost-lg + + + 2n-zn2-vpp-cov-ip4-00: + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-lg: + infra: + 25ge2p1xxv710: + - avf + - vfio-pci + - ip4-3-md: + infra: + 100ge2p1cx556a: + - rdma-core + - mlx5_core + + 2n-zn2-vpp-cov-ip4-01: + infra: + 25ge2p1xxv710: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-pol-md + + 2n-zn2-vpp-cov-ip4-02: + infra: + 25ge2p1xxv710: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-acl-lg + + 2n-zn2-vpp-cov-ip4-03: + infra: + 25ge2p1xxv710: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - nat44det-sm + + 2n-zn2-vpp-cov-ip4tun-00: + infra: + 25ge2p1xxv710: + - avf + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - geneve-lg + + 2n-zn2-vpp-cov-ip6-00: + infra: + 25ge2p1xxv710: + - avf + - vfio-pci + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-acl-md + - ip6-base-sm: + infra: + 100ge2p1cx556a: + - rdma-core + - mlx5_core + + 2n-zn2-vpp-cov-ip6-01: + infra: + 25ge2p1xxv710: + - avf + - vfio-pci + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-scale-lg + - ip6-2m-sm: + infra: + 100ge2p1cx556a: + - rdma-core + - mlx5_core + + 2n-zn2-vpp-cov-l2-00: + infra: + 25ge2p1xxv710: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-base-md + - l2-base-sm: + infra: + 100ge2p1cx556a: + - rdma-core + - mlx5_core + + 2n-zn2-vpp-cov-lb-00: + infra: + 25ge2p1xxv710: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - lb-md + + 2n-zn2-vpp-cov-memif-00: + infra: + 25ge2p1xxv710: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - container-memif-l2bd-sw-sm + - container-memif-sw-sm + + 2n-zn2-vpp-cov-vhost-00: + infra: + 25ge2p1xxv710: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - vhost-lg + + + 3n-alt-vpp-cov-ip4-00: + infra: + 100ge2p1cx6dx: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-lg + + 3n-alt-vpp-cov-ip4-01: + infra: + 100ge2p1cx6dx: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-pol-md + + 3n-alt-vpp-cov-ip4-02: + infra: + 100ge2p1cx6dx: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-acl-2-md + + 3n-alt-vpp-cov-ip4tun-00: + infra: + 100ge2p1cx6dx: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - vxlan-sm + + 3n-alt-vpp-cov-ip6-00: + infra: + 100ge2p1cx6dx: + - mlx5_core + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-acl-md + - ip6-scale-md + + 3n-alt-vpp-cov-l2-00: + infra: + 100ge2p1cx6dx: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-base-md + - l2-scale-md + + 3n-alt-vpp-cov-l2-01: + infra: + 100ge2p1cx6dx: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-acl-2-md + - l2-macip-sm + + 3n-alt-vpp-cov-memif-00: + infra: + 100ge2p1cx6dx: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - container-memif-sw-sm + + 3n-alt-vpp-cov-srv6-00: + infra: + 100ge2p1cx6dx: + - mlx5_core + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - srv6-lg + + 3n-alt-vpp-cov-vhost-00: + infra: + 100ge2p1cx6dx: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - vhost-2-md + - vhost-vxlan-2-sm + + + 3n-emr-vpp-cov-crypto-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: [64, 1518, imix] + core: [1, ] + tests: + - ipsec-sw-aes128cbc-hmac512sha-lg + - ipsec-sw-policy-aes256gcm-sm + + 3n-emr-vpp-cov-crypto-01: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - imix + tests: + - ipsec-sw-aes128gcm-lg + + 3n-emr-vpp-cov-crypto-02: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - imix + tests: + - ipsec-sw-aes256gcm-lg + + 3n-emr-vpp-cov-crypto-03: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - imix + tests: + - ipsec-sw-async-aes128cbc-hmac512sha-md + - ipsec-sw-async-aes128gcm-md + - ipsec-sw-async-aes256gcm-md + + 3n-emr-vpp-cov-crypto-04: + infra: + 100ge2p1e810cq: + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - imix + tests: + - ipsec-hw-async-aes256gcm-md + - ipsec-hw-async-policy-aes256gcm-md + + 3n-emr-vpp-cov-ip4-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-2-md + + 3n-emr-vpp-cov-ip4-01: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-pol-md + + 3n-emr-vpp-cov-ip4-02: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-acl-lg + + 3n-emr-vpp-cov-ip4tun-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - vxlan-sm + + 3n-emr-vpp-cov-ip4tun-01: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: [64, 1518, imix] + tests: + - wireguard-sw-lg + + 3n-emr-vpp-cov-ip4tun-02: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - lisp-sm + + 3n-emr-vpp-cov-ip6-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-acl-md + + 3n-emr-vpp-cov-ip6-01: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-scale-md + + 3n-emr-vpp-cov-ip6tun-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + tests: + - ip6-lisp-sm + + 3n-emr-vpp-cov-l2-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-base-md + + 3n-emr-vpp-cov-l2-01: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-scale-md + + 3n-emr-vpp-cov-l2-02: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-acl-xl + + 3n-emr-vpp-cov-l2-03: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-macip-md + + 3n-emr-vpp-cov-memif-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - container-memif-sw-sm + + 3n-emr-vpp-cov-srv6-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - srv6-lg + + 3n-emr-vpp-cov-vhost-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - vhost-2-md + - vhost-vxlan-2-sm + + + 3n-icx-vpp-cov-crypto-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: [64, 1518, imix] + core: [1, ] + tests: + - ipsec-sw-aes128cbc-hmac512sha-lg + - ipsec-sw-policy-aes256gcm-sm + + 3n-icx-vpp-cov-crypto-01: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - imix + tests: + - ipsec-sw-aes128gcm-lg + + 3n-icx-vpp-cov-crypto-02: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - imix + tests: + - ipsec-sw-aes256gcm-lg + + 3n-icx-vpp-cov-crypto-03: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - imix + tests: + - ipsec-sw-async-aes128cbc-hmac512sha-md + - ipsec-sw-async-aes128gcm-md + - ipsec-sw-async-aes256gcm-md + + 3n-icx-vpp-cov-crypto-04: + infra: + 25ge2p1e810xxv: + - vfio-pci + framesize: [64, 1518, imix] + tests: + - ipsec-sw-fixtnlip-md + + 3n-icx-vpp-cov-crypto-10: + infra: + 100ge2p1cx6dx: + - mlx5_core + framesize: [64, 1518, imix] + core: [1, ] + tests: + - ipsec-sw-aes128cbc-hmac512sha-lg + - ipsec-sw-policy-aes256gcm-sm + + 3n-icx-vpp-cov-crypto-11: + infra: + 100ge2p1cx6dx: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - imix + tests: + - ipsec-sw-aes128gcm-lg + + 3n-icx-vpp-cov-crypto-12: + infra: + 100ge2p1cx6dx: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - imix + tests: + - ipsec-sw-aes256gcm-lg + + 3n-icx-vpp-cov-crypto-13: + infra: + 100ge2p1cx6dx: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - imix + tests: + - ipsec-sw-async-aes128cbc-hmac512sha-md + - ipsec-sw-async-aes128gcm-md + - ipsec-sw-async-aes256gcm-md + + 3n-icx-vpp-cov-ip4-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-2-md + + 3n-icx-vpp-cov-ip4-01: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-pol-md + + 3n-icx-vpp-cov-ip4-02: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-acl-lg + + 3n-icx-vpp-cov-ip4-10: + infra: + 100ge2p1cx6dx: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-2-md + + 3n-icx-vpp-cov-ip4-11: + infra: + 100ge2p1cx6dx: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-pol-md + + 3n-icx-vpp-cov-ip4-12: + infra: + 100ge2p1cx6dx: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-acl-lg + + 3n-icx-vpp-cov-ip4tun-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - vxlan-sm + + 3n-icx-vpp-cov-ip4tun-01: + infra: + 100ge2p1e810cq: + - vfio-pci + framesize: [64, 1518, imix] + tests: + - wireguard-sw-lg + + 3n-icx-vpp-cov-ip4tun-02: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - lisp-sm + + 3n-icx-vpp-cov-ip4tun-03: + infra: + 100ge2p1e810cq: + - vfio-pci + 25ge2p1e810xxv: + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - gtpu-sw-sm + - gtpu-hw-sm + + 3n-icx-vpp-cov-ip4tun-10: + infra: + 100ge2p1cx6dx: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - vxlan-sm + + 3n-icx-vpp-cov-ip4tun-11: + infra: + 100ge2p1cx6dx: + - mlx5_core + framesize: [64, 1518, imix] + tests: + - wireguard-sw-lg + + 3n-icx-vpp-cov-ip4tun-12: + infra: + 100ge2p1cx6dx: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - lisp-sm + + 3n-icx-vpp-cov-ip6-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-acl-md + + 3n-icx-vpp-cov-ip6-01: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-scale-md + + 3n-icx-vpp-cov-ip6-10: + infra: + 100ge2p1cx6dx: + - mlx5_core + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-acl-md + + 3n-icx-vpp-cov-ip6-11: + infra: + 100ge2p1cx6dx: + - mlx5_core + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-scale-md + + 3n-icx-vpp-cov-ip6tun-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-lisp-sm + + 3n-icx-vpp-cov-ip6tun-10: + infra: + 100ge2p1cx6dx: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-lisp-sm + + 3n-icx-vpp-cov-l2-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-base-md + + 3n-icx-vpp-cov-l2-01: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-scale-md + + 3n-icx-vpp-cov-l2-02: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-acl-xl + + 3n-icx-vpp-cov-l2-03: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-macip-md + + 3n-icx-vpp-cov-l2-10: + infra: + 100ge2p1cx6dx: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-base-md + + 3n-icx-vpp-cov-l2-11: + infra: + 100ge2p1cx6dx: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-scale-md + + 3n-icx-vpp-cov-l2-12: + infra: + 100ge2p1cx6dx: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-acl-xl + + 3n-icx-vpp-cov-l2-13: + infra: + 100ge2p1cx6dx: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-macip-md + + 3n-icx-vpp-cov-memif-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - container-memif-sw-sm + + 3n-icx-vpp-cov-memif-10: + infra: + 100ge2p1cx6dx: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - container-memif-sw-sm + + 3n-icx-vpp-cov-srv6-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - srv6-lg + + 3n-icx-vpp-cov-srv6-10: + infra: + 100ge2p1cx6dx: + - mlx5_core + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - srv6-lg + + 3n-icx-vpp-cov-vhost-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - vhost-2-md + - vhost-vxlan-2-sm + + 3n-icx-vpp-cov-vhost-10: + infra: + 100ge2p1cx6dx: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - vhost-2-md + - vhost-vxlan-2-sm + + + 3n-icxd-vpp-cov-crypto-00: + infra: + 25ge2p1e823c: + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - imix + tests: + - ipsec-sw-aes128cbc-hmac512sha-md + - ipsec-sw-aes128gcm-md + - ipsec-sw-aes256gcm-md + - ipsec-sw-policy-aes256gcm-sm + + 3n-icxd-vpp-cov-crypto-01: + infra: + 25ge2p1e823c: + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - imix + tests: + - ipsec-sw-async-aes128cbc-hmac512sha-md + - ipsec-sw-async-aes128gcm-md + - ipsec-sw-async-aes256gcm-md + + 3n-icxd-vpp-cov-ip4-00: + infra: + 25ge2p1e823c: + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-lg + + 3n-icxd-vpp-cov-ip4-01: + infra: + 25ge2p1e823c: + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-pol-md + + 3n-icxd-vpp-cov-ip4tun-00: + infra: + 25ge2p1e823c: + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - vxlan-sm + + 3n-icxd-vpp-cov-ip4tun-01: + infra: + 25ge2p1e823c: + - vfio-pci + framesize: [64, 1518, imix] + tests: + - wireguard-sw-lg + - wireguard-hw-lg + + 3n-icxd-vpp-cov-ip6-00: + infra: + 25ge2p1e823c: + - vfio-pci + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-acl-md + + 3n-icxd-vpp-cov-ip6-01: + infra: + 25ge2p1e823c: + - vfio-pci + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-scale-md + + 3n-icxd-vpp-cov-l2-00: + infra: + 25ge2p1e823c: + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-base-md + + 3n-icxd-vpp-cov-l2-01: + infra: + 25ge2p1e823c: + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-scale-md + + + 3n-oct-vpp-cov-ip4-00: + infra: + 100ge2p1a063: + - oct-vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-aws-md + + 3n-oct-vpp-cov-l2-00: + infra: + 100ge2p1a063: + - oct-vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-md + + + 3n-snr-vpp-cov-crypto-00: + infra: + 25ge2p1e822cq: + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - imix + tests: + - ipsec-sw-aes128cbc-hmac512sha-md + - ipsec-sw-aes128gcm-md + - ipsec-sw-aes256gcm-md + - ipsec-sw-policy-aes256gcm-sm + + 3n-snr-vpp-cov-crypto-01: + infra: + 25ge2p1e822cq: + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - imix + tests: + - ipsec-sw-async-aes128cbc-hmac512sha-md + - ipsec-sw-async-aes128gcm-md + - ipsec-sw-async-aes256gcm-md + + 3n-snr-vpp-cov-ip4-00: + infra: + 25ge2p1e822cq: + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-lg + + 3n-snr-vpp-cov-ip4-01: + infra: + 25ge2p1e822cq: + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-pol-md + + 3n-snr-vpp-cov-ip4tun-00: + infra: + 25ge2p1e822cq: + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - vxlan-sm + + 3n-snr-vpp-cov-ip4tun-01: + infra: + 25ge2p1e822cq: + - vfio-pci + framesize: [64, 1518, imix] + tests: + - wireguard-sw-lg + - wireguard-hw-lg + + 3n-snr-vpp-cov-ip6-00: + infra: + 25ge2p1e822cq: + - vfio-pci + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-acl-md + + 3n-snr-vpp-cov-ip6-01: + infra: + 25ge2p1e822cq: + - vfio-pci + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-scale-md + + 3n-snr-vpp-cov-l2-00: + infra: + 25ge2p1e822cq: + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-base-md + + 3n-snr-vpp-cov-l2-01: + infra: + 25ge2p1e822cq: + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-scale-md + + + 3na-spr-vpp-cov-crypto-01: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - imix + tests: + - ipsec-sw-aes128gcm-lg + + 3na-spr-vpp-cov-crypto-02: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - imix + tests: + - ipsec-sw-aes256gcm-lg + + 3na-spr-vpp-cov-crypto-03: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - imix + tests: + - ipsec-sw-async-aes128cbc-hmac512sha-md + - ipsec-sw-async-aes128gcm-md + - ipsec-sw-async-aes256gcm-md + + 3na-spr-vpp-cov-crypto-04: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: [64, 1518, imix] + tests: + - ipsec-sw-fixtnlip-md + + 3na-spr-vpp-cov-ip4-00: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-2-md + - ip4-60k-sm: + infra: + 200ge6p3cx7veat: + - mlx5_core + framesize: [64, 1518, imix] + core: [3, 6, 12] + + 3na-spr-vpp-cov-ip4-01: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-pol-md + + 3na-spr-vpp-cov-ip4-02: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-acl-lg + + 3na-spr-vpp-cov-ip4tun-00: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - vxlan-sm + + 3na-spr-vpp-cov-ip4tun-01: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: [64, 1518, imix] + tests: + - wireguard-sw-lg + + 3na-spr-vpp-cov-ip4tun-02: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - lisp-sm + + 3na-spr-vpp-cov-ip4tun-03: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - gtpu-sw-sm + + 3na-spr-vpp-cov-ip6-00: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-acl-md + + 3na-spr-vpp-cov-ip6-01: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-scale-md + + 3na-spr-vpp-cov-ip6tun-00: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-lisp-sm + + 3na-spr-vpp-cov-l2-00: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-base-md + + 3na-spr-vpp-cov-l2-01: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-scale-md + + 3na-spr-vpp-cov-l2-02: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-acl-xl + + 3na-spr-vpp-cov-l2-03: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-macip-md + + 3na-spr-vpp-cov-memif-00: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - container-memif-sw-sm + + 3na-spr-vpp-cov-srv6-00: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - srv6-lg + + 3na-spr-vpp-cov-vhost-00: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - vhost-2-md + - vhost-vxlan-2-sm + + + 3nb-spr-vpp-cov-crypto-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: [64, 1518, imix] + core: [1, ] + tests: + - ipsec-sw-aes128cbc-hmac512sha-lg + - ipsec-sw-policy-aes256gcm-sm + + 3nb-spr-vpp-cov-crypto-01: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - imix + tests: + - ipsec-sw-aes128gcm-lg + + 3nb-spr-vpp-cov-crypto-02: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - imix + tests: + - ipsec-sw-aes256gcm-lg + + 3nb-spr-vpp-cov-crypto-03: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - imix + tests: + - ipsec-sw-async-aes128cbc-hmac512sha-md + - ipsec-sw-async-aes128gcm-md + - ipsec-sw-async-aes256gcm-md + + 3nb-spr-vpp-cov-crypto-04: + infra: + 25ge2p1e810xxv: + - vfio-pci + framesize: [64, 1518, imix] + tests: + - ipsec-sw-fixtnlip-md + + 3nb-spr-vpp-cov-crypto-05: + infra: + 100ge2p1e810cq: + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - imix + tests: + - ipsec-hw-async-aes256gcm-md + - ipsec-hw-async-policy-aes256gcm-md + + 3nb-spr-vpp-cov-ip4-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-2-md + + 3nb-spr-vpp-cov-ip4-01: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-pol-md + + 3nb-spr-vpp-cov-ip4-02: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip4-acl-lg + + 3nb-spr-vpp-cov-ip4tun-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + 25ge2p1e810xxv: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - vxlan-sm + + 3nb-spr-vpp-cov-ip4tun-01: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: [64, 1518, imix] + tests: + - wireguard-sw-lg + + 3nb-spr-vpp-cov-ip4tun-02: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - lisp-sm + + 3nb-spr-vpp-cov-ip4tun-03: + infra: + 100ge2p1e810cq: + - vfio-pci + 25ge2p1e810xxv: + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - gtpu-sw-sm + - gtpu-hw-sm + + 3nb-spr-vpp-cov-ip4tun-04: + infra: + 100ge2p1e810cq: + - vfio-pci + framesize: [64, 1518, imix] + tests: + - wireguard-hw-lg + + 3nb-spr-vpp-cov-ip6-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-acl-md + + 3nb-spr-vpp-cov-ip6-01: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-scale-md + + 3nb-spr-vpp-cov-ip6-02: + infra: + 25ge2p1e810xxv: + - avf + - vfio-pci + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-base-sm + + 3nb-spr-vpp-cov-ip6tun-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-lisp-sm + + 3nb-spr-vpp-cov-l2-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + 25ge2p1e810xxv: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-base-md + + 3nb-spr-vpp-cov-l2-01: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-scale-md + + 3nb-spr-vpp-cov-l2-02: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-acl-xl + + 3nb-spr-vpp-cov-l2-03: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - l2-macip-md + + 3nb-spr-vpp-cov-memif-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - container-memif-sw-sm + + 3nb-spr-vpp-cov-srv6-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - srv6-lg + + 3nb-spr-vpp-cov-srv6-10: + infra: + 25ge2p1e810xxv: + - avf + - vfio-pci + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - srv6-lg + + 3nb-spr-vpp-cov-vhost-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - vhost-2-md + - vhost-vxlan-2-sm + + 3nb-spr-vpp-cov-vhost-10: + infra: + 25ge2p1e810xxv: + - avf + - vfio-pci + framesize: + - 64 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - vhost-2-md + - vhost-vxlan-2-sm + + + 2n-aws-vpp-iterative: + infra: + 50ge1p1ena: + - vfio-pci + framesize: [64, 1518] + core: [1, 2] + tests: + - ip4-aws-md + - ip6-aws-md: + framesize: [78, 1518] + + 2n-c6in-vpp-iterative: + infra: + 200ge1p1ena: + - vfio-pci + framesize: [64, 1518] + core: [1, 2] + tests: + - ip4-aws-md + - ip6-aws-md: + framesize: [78, 1518] + + 2n-c7gn-vpp-iterative: + infra: + 100ge1p1ena: + - vfio-pci + framesize: [64, 1518] + core: [1, 2] + tests: + - ip4-aws-md + - ip6-aws-md: + framesize: [78, 1518] + + 2n-emr-vpp-iterative: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + tests: + - container-memif-l2bd-sw-sm: + framesize: [64, 1518] + - container-memif-hw-sm: + framesize: [64, 1518] + - container-memif-sw-sm + - ip4-md + - ip4-acl-md: + infra: + 100ge2p1e810cq: + - avf + - nat44det-sm: + infra: + 100ge2p1e810cq: + - avf + - nat44-udir-sm: + infra: + 100ge2p1e810cq: + - avf + - nat44ed-cps-sm: + infra: + 100ge2p1e810cq: + - avf + - nat44ed-tput-sm: + infra: + 100ge2p1e810cq: + - avf + framesize: [100, ] + - nat44-ip4base-cps-sm: + infra: + 100ge2p1e810cq: + - avf + - nat44-ip4base-tput-sm: + infra: + 100ge2p1e810cq: + - avf + framesize: [100, ] + - geneve-md: + infra: + 100ge2p1e810cq: + - avf + - ip6-md: + framesize: [78, ] + - l2-lg + - vhost-md + + 2n-grc-vpp-iterative: + infra: + 200ge2p1cx7veat: + - mlx5_core + tests: + - container-memif-l2bd-sw-sm + - container-memif-sw-sm + - ip4-md + - ip4-acl-md + - nat44det-sm + - nat44-udir-sm + - nat44ed-cps-sm + - nat44ed-tput-sm: + framesize: [100, ] + - nat44-ip4base-cps-sm + - nat44-ip4base-tput-sm: + framesize: [100, ] + - geneve-md + - ip6-md: + framesize: [78, ] + - l2-lg + - vhost-md + + 2n-icx-vpp-iterative: + infra: + 200ge2p1cx7veat: + - mlx5_core + 100ge2p1e810cq: + - avf + - vfio-pci + 25ge2p1e810xxv: + - avf + tests: + - container-memif-l2bd-sw-sm + - container-memif-sw-sm + - ip4-md + - ip4-acl-md: + infra: + 200ge2p1cx7veat: + - mlx5_core + 100ge2p1e810cq: + - avf + - nat44det-sm: + infra: + 200ge2p1cx7veat: + - mlx5_core + 100ge2p1e810cq: + - avf + - nat44-udir-sm: + infra: + 200ge2p1cx7veat: + - mlx5_core + 100ge2p1e810cq: + - avf + - nat44ed-cps-sm: + infra: + 200ge2p1cx7veat: + - mlx5_core + 100ge2p1e810cq: + - avf + - nat44ed-tput-sm: + infra: + 200ge2p1cx7veat: + - mlx5_core + 100ge2p1e810cq: + - avf + framesize: [100, ] + - nat44-ip4base-cps-sm: + infra: + 200ge2p1cx7veat: + - mlx5_core + 100ge2p1e810cq: + - avf + - nat44-ip4base-tput-sm: + infra: + 200ge2p1cx7veat: + - mlx5_core + 100ge2p1e810cq: + - avf + framesize: [100, ] + - geneve-md: + infra: + 200ge2p1cx7veat: + - mlx5_core + 100ge2p1e810cq: + - avf + - ip6-md: + framesize: [78, ] + - l2-lg + - vhost-md + + 2n-spr-vpp-iterative: + infra: + 200ge2p1cx7veat: + - mlx5_core + 100ge2p1e810cq: + - avf + - vfio-pci + 25ge2p1e810xxv: + - avf + tests: + - container-memif-l2bd-sw-sm: + framesize: [64, 1518] + - container-memif-sw-sm + - container-memif-hw-sm: + framesize: [64, 1518] + - ip4-md + - ip4-acl-md: + infra: + 200ge2p1cx7veat: + - mlx5_core + 100ge2p1e810cq: + - avf + - nat44det-sm: + infra: + 200ge2p1cx7veat: + - mlx5_core + 100ge2p1e810cq: + - avf + - nat44-udir-sm: + infra: + 200ge2p1cx7veat: + - mlx5_core + 100ge2p1e810cq: + - avf + - nat44ed-cps-sm: + infra: + 200ge2p1cx7veat: + - mlx5_core + 100ge2p1e810cq: + - avf + - nat44ed-tput-sm: + infra: + 200ge2p1cx7veat: + - mlx5_core + 100ge2p1e810cq: + - avf + framesize: [100, ] + - nat44-ip4base-cps-sm: + infra: + 200ge2p1cx7veat: + - mlx5_core + 100ge2p1e810cq: + - avf + - nat44-ip4base-tput-sm: + infra: + 200ge2p1cx7veat: + - mlx5_core + 100ge2p1e810cq: + - avf + framesize: [100, ] + - geneve-md: + infra: + 200ge2p1cx7veat: + - mlx5_core + 100ge2p1e810cq: + - avf + - ip6-md: + framesize: [78, ] + - l2-lg + - vhost-md + + 2n-zn2-vpp-iterative: + infra: + 25ge2p1xxv710: + - avf + - vfio-pci + 100ge2p1cx556a: + - rdma-core + tests: + - container-memif-l2bd-sw-sm + - container-memif-sw-sm + - ip4-aws-md + - ip4-acl-md: + infra: + 25ge2p1xxv710: + - avf + 100ge2p1cx556a: + - rdma-core + - nat44det-sm: + infra: + 25ge2p1xxv710: + - avf + - geneve-lg: + infra: + 25ge2p1xxv710: + - avf + - ip6-aws-md: + framesize: [78, ] + - l2-lg + - vhost-md + + 3n-alt-vpp-iterative: + infra: + 100ge2p1cx6dx: + - mlx5_core + tests: + - ipsec-sw-async-md: + framesize: [1518, imix] + core: [2, 3, 4] + - ipsec-sw-md: + framesize: [1518, imix] + - ipsec-sw-reassembly-sm: + framesize: [1518, ] + - ipsec-sw-policy-md: + framesize: [1518, imix] + - ip4-base-sm + - ip4-pol-sm + - vxlan-sm + - gtpu-sw-sm + - wireguard-sw-md: + framesize: [1518, imix] + - gtpu-sw-reassembly-sm: + framesize: [1518, ] + - ip6-base-sm: + framesize: [78, ] + - l2-base-md + - srv6-lg: + framesize: [78, ] + - vhost-2-md + + 3n-emr-vpp-iterative: + infra: + 100ge2p1e810cq: + - vfio-pci + tests: + - ipsec-sw-lg: + infra: + 100ge2p1e810cq: + - avf + framesize: [1518, imix] + - ipsec-sw-async-md: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: [1518, imix] + core: [2, 3, 4] + - ipsec-sw-md: + framesize: [1518, imix] + - ipsec-hw-md: + framesize: [1518, imix] + - ipsec-sw-reassembly-sm: + framesize: [1518, ] + - ipsec-sw-udp: + framesize: [1518, ] + core: [1, ] + - ip4-base-sm: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + - vxlan-sm: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + - gtpu-sw-sm + - gtpu-sw-reassembly-sm: + framesize: [1518, ] + - gtpu-hw-sm + - wireguard-sw-md: + framesize: [1518, imix] + - wireguard-hw-md: + framesize: [1518, imix] + - ip6-base-sm: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: [78, ] + - l2-base-md: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + - srv6-lg: + infra: + 100ge2p1e810cq: + - avf + framesize: [78, ] + + 3n-icx-vpp-iterative: + tests: + - ipsec-sw-md: + infra: + 100ge2p1e810cq: + - vfio-pci + framesize: [1518, imix] + - ipsec-sw-lg: + infra: + 100ge2p1cx6dx: + - mlx5_core + framesize: [1518, imix] + - ipsec-sw-lg: + infra: + 100ge2p1e810cq: + - avf + framesize: [1518, imix] + core: [1, ] + - ipsec-sw-async-md: + infra: + 100ge2p1cx6dx: + - mlx5_core + 100ge2p1e810cq: + - avf + framesize: [1518, imix] + core: [2, 3, 4] + - ipsec-sw-policy-md: + infra: + 25ge2p1e810xxv: + - vfio-pci + framesize: [1518, imix] + - ipsec-sw-udp: + infra: + 100ge2p1e810cq: + - vfio-pci + framesize: [1518, ] + core: [1, ] + - ipsec-sw-async-sm: + infra: + 100ge2p1e810cq: + - vfio-pci + framesize: [1518, imix] + - ipsec-sw-reassembly-sm: + infra: + 100ge2p1e810cq: + - vfio-pci + framesize: [1518, ] + - ip4-base-sm: + infra: + 100ge2p1cx6dx: + - mlx5_core + 100ge2p1e810cq: + - avf + - vfio-pci + - gtpu-sw-sm: + infra: + 100ge2p1cx6dx: + - mlx5_core + 25ge2p1e810xxv: + - vfio-pci + 100ge2p1e810cq: + - vfio-pci + - gtpu-hw-sm: + infra: + 25ge2p1e810xxv: + - vfio-pci + 100ge2p1e810cq: + - vfio-pci + - gtpu-sw-reassembly-sm: + infra: + 100ge2p1e810cq: + - vfio-pci + framesize: [1518, ] + - wireguard-sw-sm: + infra: + 100ge2p1cx6dx: + - mlx5_core + 25ge2p1e810xxv: + - vfio-pci + - wireguard-sw-lg: + infra: + 100ge2p1e810cq: + - vfio-pci + framesize: [1518, imix] + - vxlan-sm: + infra: + 100ge2p1e810cq: + - vfio-pci + - avf + - ip6-base-sm: + infra: + 100ge2p1cx6dx: + - mlx5_core + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: [78, ] + - l2-base-md: + infra: + 100ge2p1cx6dx: + - mlx5_core + 100ge2p1e810cq: + - avf + - vfio-pci + - l2-1lbvpplacp-sm: + infra: + 100ge2p1cx6dx: + - mlx5_core + 100ge2p1e810cq: + - avf + - srv6-lg: + infra: + 100ge2p1cx6dx: + - mlx5_core + 100ge2p1e810cq: + - avf + framesize: [78, ] + + 3n-icxd-vpp-iterative: + infra: + 25ge2p1e823c: + - vfio-pci + framesize: [1518, imix] + tests: + - ipsec-sw-md + - ipsec-sw-udp: + framesize: [1518, ] + core: [1, ] + - ipsec-sw-async-2-sm: + core: [2, 3, 4] + - ipsec-sw-reassembly-sm: + framesize: [1518, ] + - ipsec-hw-sm + - ip4-aws-md: + framesize: [64, ] + - vxlan-sm: + framesize: [64, ] + - wireguard-sw-lg + - wireguard-hw-lg + - gtpu-sw-reassembly-sm: + framesize: [1518, ] + - ip6-aws-md: + framesize: [78, ] + - l2-lg: + framesize: [64, ] + + 3n-oct-vpp-iterative: + infra: + 100ge2p1a063: + - oct-vfio-pci + tests: + - ip4-aws-md + - l2-md + + 3n-snr-vpp-iterative: + infra: + 25ge2p1e822cq: + - vfio-pci + framesize: [1518, imix] + tests: + - ipsec-sw-md + - ipsec-sw-udp: + framesize: [1518, ] + core: [1, ] + - ipsec-sw-async-2-sm: + core: [2, 3, 4] + - ipsec-sw-reassembly-sm: + framesize: [1518, ] + - ipsec-hw-sm + - ip4-aws-md: + framesize: [64, ] + - vxlan-sm: + framesize: [64, ] + - wireguard-sw-lg + - wireguard-hw-lg + - gtpu-sw-reassembly-sm: + framesize: [1518, ] + - ip6-aws-md: + framesize: [78, ] + - l2-lg: + framesize: [64, ] + + 3na-spr-vpp-iterative: + infra: + 200ge2p1cx7veat: + - mlx5_core + framesize: [1518, imix] + tests: + - ipsec-sw-lg + - ipsec-sw-udp: + framesize: [1518, ] + core: [1, ] + - ipsec-sw-async-md: + core: [2, 3, 4] + - ip4-2-md: + framesize: [64, ] + - ip4-60k-sm: + infra: + 200ge6p3cx7veat: + - mlx5_core + framesize: [64, 1518, imix] + core: [3, 6, 12] + - vxlan-sm: + framesize: [64, ] + - wireguard-sw-lg + - gtpu-sw-sm: + framesize: [64, ] + - ip6-base-sm: + framesize: [78, ] + - l2-base-md: + framesize: [64, ] + - srv6-lg: + framesize: [78, ] + + 3nb-spr-vpp-iterative: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + 25ge2p1e810xxv: + - avf + - vfio-pci + framesize: [1518, imix] + tests: + - ipsec-sw-lg: + infra: + 100ge2p1e810cq: + - avf + - ipsec-sw-async-md: + core: [2, 3, 4] + infra: + 100ge2p1e810cq: + - avf + - ipsec-sw-md: + infra: + 100ge2p1e810cq: + - vfio-pci + - ipsec-sw-udp: + infra: + 100ge2p1e810cq: + - vfio-pci + framesize: [1518, ] + core: [1, ] + - ipsec-sw-async-sm: + core: [2, 3, 4] + infra: + 100ge2p1e810cq: + - vfio-pci + - ipsec-hw-md: + infra: + 100ge2p1e810cq: + - vfio-pci + - ipsec-sw-reassembly-sm: + infra: + 100ge2p1e810cq: + - vfio-pci + framesize: [1518, ] + - ip4-base-sm: + framesize: [64, ] + - vxlan-sm: + framesize: [64, ] + - gtpu-sw-sm: + infra: + 100ge2p1e810cq: + - vfio-pci + 25ge2p1e810xxv: + - avf + - vfio-pci + framesize: [64, ] + - wireguard-sw-sm: + infra: + 25ge2p1e810xxv: + - vfio-pci + framesize: [64, ] + - gtpu-hw-sm: + infra: + 100ge2p1e810cq: + - vfio-pci + 25ge2p1e810xxv: + - vfio-pci + framesize: [64, ] + - gtpu-sw-reassembly-sm: + infra: + 100ge2p1e810cq: + - vfio-pci + framesize: [1518, ] + - wireguard-sw-md: + infra: + 100ge2p1e810cq: + - vfio-pci + - wireguard-hw-md: + infra: + 100ge2p1e810cq: + - vfio-pci + - ip6-base-sm: + framesize: [78, ] + - l2-base-md: + infra: + 100ge2p1e810cq: + - avf + 25ge2p1e810xxv: + - avf + framesize: [64, ] + - l2-sm: + infra: + 100ge2p1e810cq: + - vfio-pci + 25ge2p1e810xxv: + - vfio-pci + framesize: [64, ] + - srv6-lg: + infra: + 100ge2p1e810cq: + - avf + framesize: [78, ] + + + 2n-icx-vpp-soak: + core: [1, ] + infra: + 100ge2p1e810cq: + - avf + tests: + - soak-memif-sm: + framesize: [64, 1518] + - soak-ip4-sm: + infra: + 200ge2p1cx7veat: + - mlx5_core + 100ge2p1e810cq: + - vfio-pci + - soak-ip4-md + - soak-nat44-cps-sm + - soak-nat44-tput-sm: + framesize: [100, ] + - soak-ip6-sm: + infra: + 200ge2p1cx7veat: + - mlx5_core + 100ge2p1e810cq: + - vfio-pci + framesize: [78, ] + - soak-ip6-md: + framesize: [78, ] + - soak-l2-sm: + infra: + 200ge2p1cx7veat: + - mlx5_core + 100ge2p1e810cq: + - vfio-pci + - soak-l2-md + - soak-vhost-sm + + 2n-spr-vpp-soak: + core: [1, ] + infra: + 100ge2p1e810cq: + - avf + tests: + - soak-memif-sm: + framesize: [64, 1518] + - soak-memif-dma-sm: + framesize: [64, 1518] + - soak-ip4-sm: + infra: + 200ge2p1cx7veat: + - mlx5_core + 100ge2p1e810cq: + - vfio-pci + - soak-ip4-md + - soak-nat44-cps-sm + - soak-nat44-tput-sm: + framesize: [100, ] + - soak-ip6-sm: + infra: + 200ge2p1cx7veat: + - mlx5_core + 100ge2p1e810cq: + - vfio-pci + framesize: [78, ] + - soak-ip6-md: + framesize: [78, ] + - soak-l2-sm: + infra: + 200ge2p1cx7veat: + - mlx5_core + 100ge2p1e810cq: + - vfio-pci + - soak-l2-md + - soak-vhost-sm + + 3n-icx-vpp-soak: + core: [1, ] + infra: + 100ge2p1e810cq: + - avf + tests: + # - soak-memif-sm: + # framesize: [64, 1518] + - soak-crypto-sm: + infra: + 100ge2p1e810cq: + - vfio-pci + framesize: [1518, ] + - soak-ip4-sm: + infra: + 100ge2p1cx6dx: + - mlx5_core + 100ge2p1e810cq: + - vfio-pci + - soak-ip4-md + - soak-ip6-sm: + infra: + 100ge2p1cx6dx: + - mlx5_core + 100ge2p1e810cq: + - vfio-pci + framesize: [78, ] + - soak-ip6-md: + framesize: [78, ] + - soak-l2-sm: + infra: + 100ge2p1cx6dx: + - mlx5_core + 100ge2p1e810cq: + - vfio-pci + - soak-l2-md + - soak-srv6-sm: + framesize: [78, ] + - soak-vhost-sm + + 3n-icxd-vpp-soak: + core: [1, ] + infra: + 25ge2p1e823c: + - avf + tests: + # - soak-memif-sm: + # framesize: [64, 1518] + - soak-crypto-md: + infra: + 25ge2p1e823c: + - vfio-pci + framesize: [1518, ] + - soak-ip4-sm: + infra: + 100ge2p1cx6dx: + - mlx5_core + 25ge2p1e823c: + - vfio-pci + - soak-ip4-md + - soak-ip6-sm: + infra: + 100ge2p1cx6dx: + - mlx5_core + 25ge2p1e823c: + - vfio-pci + framesize: [78, ] + - soak-ip6-md: + framesize: [78, ] + - soak-l2-sm: + infra: + 100ge2p1cx6dx: + - mlx5_core + 100ge2p1e810cq: + - vfio-pci + - soak-l2-md + - soak-vhost-sm + + + 2n-emr-gso: + framesize: [128000, ] + core: [1, 2, 4] + tests: + - gso-tap-sm: + infra: + 100ge2p1e810cq: + - tap + - gso-vhost-sm: + infra: + 100ge2p1e810cq: + - vhost + + 2n-icx-gso: + framesize: [128000, ] + core: [1, 2, 4] + tests: + - gso-tap-sm: + infra: + 100ge2p1e810cq: + - tap + - gso-vhost-sm: + infra: + 100ge2p1e810cq: + - vhost + + 2n-spr-gso: + framesize: [128000, ] + core: [1, 2, 4] + tests: + - gso-tap-sm: + infra: + 100ge2p1e810cq: + - tap + - gso-vhost-sm: + infra: + 100ge2p1e810cq: + - vhost + + 2n-zn2-gso: + framesize: [128000, ] + core: [1, 2, 4] + tests: + - gso-tap-sm: + infra: + 25ge2p1e810xxv: + - tap + - gso-vhost-sm: + infra: + 25ge2p1e810xxv: + - vhost + + + 2n-icx-nfv: + framesize: [imix, ] + core: [1, 2, 4] + infra: + 25ge2p1xxv710: [avf, ] + tests: + - nfv-dcr-memif-chain-lg + - nfv-dcr-memif-pipeline-lg + - nfv-vm-vhost-chain-lg + - nfv-vm-vhost-chain-vxlan-md diff --git a/resources/libraries/bash/entry/bootstrap_verify_perf.sh b/resources/libraries/bash/entry/bootstrap_verify_perf.sh index 18dfd08c39..5cffb03166 100755 --- a/resources/libraries/bash/entry/bootstrap_verify_perf.sh +++ b/resources/libraries/bash/entry/bootstrap_verify_perf.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Cisco and/or its affiliates. +# Copyright (c) 2025 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -41,11 +41,11 @@ select_arch_os || die gather_build || die check_download_dir || die activate_virtualenv || die -generate_tests || die -archive_tests || die prepare_topology || die select_topology || die reserve_and_cleanup_testbed || die +generate_tests || die +archive_tests || die run_robot || die move_archives || die untrap_and_unreserve_testbed || die diff --git a/resources/libraries/bash/entry/tox/autogen.sh b/resources/libraries/bash/entry/tox/autogen.sh index 40cc2c2066..dc9e2bd0de 100644 --- a/resources/libraries/bash/entry/tox/autogen.sh +++ b/resources/libraries/bash/entry/tox/autogen.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2022 Cisco and/or its affiliates. +# Copyright (c) 2025 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -18,17 +18,6 @@ set -exuo pipefail # This file does not have executable flag nor shebang, # to dissuade non-tox callers. -# This script starts with copying ${CSIT_DIR}/tests to ${GENERATED_DIR}/. -# Then the script runs every executable *.py script anywhere in the copied dir, -# the working directory temporarily changed to where the *.py file is. -# Proper virtualenv is assumed to be active. -# Then another directory in ${GENERATED_DIR} is created, where -# the just generated content is copied and then overwitten by the non-generated. -# If "diff -dur" sees any changes by the overwrite, this script fails. -# The diff output is stored to autogen.log (overwriting). -# The executed *.py files are assumed to be robot suite generators, -# any change means the contribution is not consistent with the regenerated code. - # "set -eu" handles failures from the following two lines. BASH_CHECKS_DIR="$(dirname $(readlink -e "${BASH_SOURCE[0]}"))" BASH_FUNCTION_DIR="$(readlink -e "${BASH_CHECKS_DIR}/../../function")" @@ -40,23 +29,10 @@ common_dirs work_dir="$(pwd)" || die trap "cd '${work_dir}'" EXIT || die -generate_tests - -rm -rf "${GENERATED_DIR}/tests_tmp" -cp -r "${GENERATED_DIR}/tests" "${GENERATED_DIR}/tests_tmp" -# Default cp behavior is to put inside a targed dir, not to override. -cp -rf "${CSIT_DIR}/tests"/* "${GENERATED_DIR}/tests_tmp"/ -# TODO: Do we want to archive ${GENERATED_DIR}? -# I think archiving the diff is enough. - -diff_cmd=("diff" "-dur" "${GENERATED_DIR}/tests_tmp" "${GENERATED_DIR}/tests") -# Diff returns RC=1 if output is nonzero. -lines="$("${diff_cmd[@]}" | tee "autogen.log" | wc -l || true)" -if [ "${lines}" != "0" ]; then - # TODO: Decide which text goes to stdout and which to stderr. - warn "Autogen conflict, diff sees nonzero lines: ${lines}" - # TODO: Disable if output size does more harm than good. - cat "autogen.log" >&2 +get_test_code +RET_VAL="PASS" +OUTPUT=$(generate_tests) || RET_VAL="FAIL" +if [ "${RET_VAL}" == "FAIL" ]; then warn warn "Autogen checker: FAIL" exit 1 diff --git a/resources/libraries/bash/function/common.sh b/resources/libraries/bash/function/common.sh index fa8753320c..6745ad0b88 100644 --- a/resources/libraries/bash/function/common.sh +++ b/resources/libraries/bash/function/common.sh @@ -267,9 +267,8 @@ function compose_robot_arguments () { # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed. # - DUT - CSIT test/ subdirectory, set while processing tags. # - TAGS - Array variable holding selected tag boolean expressions. - # - TOPOLOGIES_TAGS - Tag boolean expression filtering tests for topology. # - TEST_CODE - The test selection string from environment or argument. - # - SELECTION_MODE - Selection criteria [test, suite, include, exclude]. + # - SELECTION_MODE - Selection criteria [none, tags]. # Variables set: # - ROBOT_ARGS - String holding part of all arguments for robot. # - EXPANDED_TAGS - Array of strings robot arguments compiled from tags. @@ -296,19 +295,15 @@ function compose_robot_arguments () { EXPANDED_TAGS=() for tag in "${TAGS[@]}"; do if [[ ${tag} == "!"* ]]; then - EXPANDED_TAGS+=("--exclude" "${tag#$"!"}") + if [ -n "${SELECTION_MODE}" ]; then + EXPANDED_TAGS+=("--exclude" "${tag#$"!"}") + fi else - if [[ ${SELECTION_MODE} == "--test" ]]; then - EXPANDED_TAGS+=("--test" "${tag}") - else - EXPANDED_TAGS+=("--include" "${TOPOLOGIES_TAGS}AND${tag}") + if [ -n "${SELECTION_MODE}" ]; then + EXPANDED_TAGS+=("--include" "${tag}") fi fi done - - if [[ ${SELECTION_MODE} == "--test" ]]; then - EXPANDED_TAGS+=("--include" "${TOPOLOGIES_TAGS}") - fi } @@ -400,19 +395,17 @@ function generate_tests () { set -exuo pipefail rm -rf "${GENERATED_DIR}/tests" || die - cp -r "${CSIT_DIR}/tests" "${GENERATED_DIR}/tests" || die - cmd_line=("find" "${GENERATED_DIR}/tests" "-type" "f") - cmd_line+=("-executable" "-name" "*.py") - # We sort the directories, so log output can be compared between runs. - file_list=$("${cmd_line[@]}" | sort) || die - - for gen in ${file_list}; do - directory="$(dirname "${gen}")" || die - filename="$(basename "${gen}")" || die - pushd "${directory}" || die - ./"${filename}" || die - popd || die - done + pushd "${CSIT_DIR}/resources/libraries/python/suite_generator" || die + + suite_gen_params=("--job" "${TEST_CODE}") + suite_gen_params+=("--gen-tests-dir" "${GENERATED_DIR}") + suite_gen_params+=("--trigger-params" "${TEST_TAG_STRING-}") + TEST_TAG_STRING=$( + for itm in ${TEST_TAG_STRING-}; do + [[ $itm == \#* ]] || printf "%s " "$itm"; + done + ) + ./suite_generator.py "${suite_gen_params[@]}" || die "Suite gen failed." } @@ -837,7 +830,6 @@ function run_robot () { # - WORKING_TOPOLOGY - Path to topology yaml file of the reserved testbed. # - DUT - CSIT test/ subdirectory, set while processing tags. # - TAGS - Array variable holding selected tag boolean expressions. - # - TOPOLOGIES_TAGS - Tag boolean expression filtering tests for topology. # - TEST_CODE - The test selection string from environment or argument. # Variables set: # - ROBOT_ARGS - String holding part of all arguments for robot. @@ -848,7 +840,9 @@ function run_robot () { set -exuo pipefail + # Run ALL generated test suites: all_options=("--outputdir" "${ARCHIVE_DIR}" "${ROBOT_ARGS[@]}") + # Run only tests defined by tag(s) out of generated tests: all_options+=("${EXPANDED_TAGS[@]}") pushd "${CSIT_DIR}" || die "Change directory operation failed." @@ -930,265 +924,47 @@ function select_tags () { # - BASH_FUNCTION_DIR - Directory with input files to process. # Variables set: # - TAGS - Array of processed tag boolean expressions. - # - SELECTION_MODE - Selection criteria [test, suite, include, exclude]. + # - SELECTION_MODE - Selection criteria [tags, empty]. set -exuo pipefail - # NIC SELECTION - case "${TEST_CODE}" in - *"1n-aws"* | *"1n-c6in"*) - start_pattern='^ SUT:' - ;; - *) - start_pattern='^ TG:' - ;; - esac - end_pattern='^ \? \?[A-Za-z0-9]\+:' - # Remove the sections from topology file - sed_command="/${start_pattern}/,/${end_pattern}/d" - # All topologies NICs - available=$(sed "${sed_command}" "${TOPOLOGIES_DIR}"/* \ - | grep -hoP "model: \K.*" | sort -u) - # Selected topology NICs - reserved=$(sed "${sed_command}" "${WORKING_TOPOLOGY}" \ - | grep -hoP "model: \K.*" | sort -u) - # All topologies NICs - Selected topology NICs - exclude_nics=($(comm -13 <(echo "${reserved}") <(echo "${available}"))) || { - die "Computation of excluded NICs failed." - } - - # Select default NIC tag. - case "${TEST_CODE}" in - *"3n-snr") - default_nic="nic_intel-e822cq" - ;; - *"3n-icxd") - default_nic="nic_intel-e823c" - ;; - *"3n-icx" | *"2n-icx") - default_nic="nic_intel-e810cq" - ;; - *"3na-spr") - default_nic="nic_mellanox-cx7veat" - ;; - *"3nb-spr") - default_nic="nic_intel-e810cq" - ;; - *"2n-spr") - default_nic="nic_intel-e810cq" - ;; - *"2n-zn2") - default_nic="nic_intel-xxv710" - ;; - *"3n-alt") - default_nic="nic_intel-xl710" - ;; - *"2n-grc") - default_nic="nic_mellanox-cx7veat" - ;; - *"2n-emr") - default_nic="nic_intel-e810cq" - ;; - *"3n-emr") - default_nic="nic_intel-e810cq" - ;; - *"3n-oct") - default_nic="nic_cavium-a063-100g" - ;; - *"1n-aws" | *"2n-aws" | *"3n-aws") - default_nic="nic_amazon-nitro-50g" - ;; - *"2n-c7gn" | *"3n-c7gn") - default_nic="nic_amazon-nitro-100g" - ;; - *"1n-c6in" | *"2n-c6in" | *"3n-c6in") - default_nic="nic_amazon-nitro-200g" - ;; - *"-x-2n"* | *"-x-3n"*) - default_nic="nic_intel-e810cq" - ;; - *) - default_nic="nic_intel-x710" - ;; - esac - sed_nic_sub_cmd="sed s/\${default_nic}/${default_nic}/" - awk_nics_sub_cmd="" - awk_nics_sub_cmd+='gsub("xxv710","25ge2p1xxv710");' - awk_nics_sub_cmd+='gsub("x710","10ge2p1x710");' - awk_nics_sub_cmd+='gsub("xl710","40ge2p1xl710");' - awk_nics_sub_cmd+='gsub("cx556a","100ge2p1cx556a");' - awk_nics_sub_cmd+='gsub("2p1cx7veat","200ge2p1cx7veat");' - awk_nics_sub_cmd+='gsub("6p3cx7veat","200ge6p3cx7veat");' - awk_nics_sub_cmd+='gsub("cx6dx","100ge2p1cx6dx");' - awk_nics_sub_cmd+='gsub("e810cq","100ge2p1e810cq");' - awk_nics_sub_cmd+='gsub("e822cq","25ge2p1e822cq");' - awk_nics_sub_cmd+='gsub("e823c","25ge2p1e823c");' - awk_nics_sub_cmd+='gsub("vic1227","10ge2p1vic1227");' - awk_nics_sub_cmd+='gsub("vic1385","40ge2p1vic1385");' - awk_nics_sub_cmd+='gsub("nitro-50g","50ge1p1ENA");' - awk_nics_sub_cmd+='gsub("nitro-100g","100ge1p1ENA");' - awk_nics_sub_cmd+='gsub("nitro-200g","200ge1p1ENA");' - awk_nics_sub_cmd+='gsub("cavium-50g","50ge2p1a063");' - awk_nics_sub_cmd+='gsub("cavium-100g","100ge2p1a063");' - awk_nics_sub_cmd+='gsub("virtual","1ge1p82540em");' - awk_nics_sub_cmd+='if ($9 =="drv_avf") drv="avf-";' - awk_nics_sub_cmd+='else if ($9 =="drv_rdma_core") drv ="rdma-";' - awk_nics_sub_cmd+='else if ($9 =="drv_mlx5_core") drv ="mlx5-";' - awk_nics_sub_cmd+='else if ($9 =="drv_af_xdp") drv ="af-xdp-";' - awk_nics_sub_cmd+='else drv="";' - awk_nics_sub_cmd+='if ($1 =="-") cores="";' - awk_nics_sub_cmd+='else cores=$1;' - awk_nics_sub_cmd+='print "*"$7"-" drv $11"-"$5"."$3"-" cores "-" drv $11"-"$5' - - # Tag file directory shorthand. - tfd="${JOB_SPECS_DIR}" + SELECTION_MODE="" case "${TEST_CODE}" in # Select specific performance tests based on jenkins job type variable. *"device"* ) - readarray -t test_tag_array <<< $(grep -v "#" \ - ${tfd}/vpp_device/${DUT}-${NODENESS}-${FLAVOR}.md | - awk {"$awk_nics_sub_cmd"} || echo "devicetest") || die - SELECTION_MODE="--test" ;; *"hoststack-daily"* ) - readarray -t test_tag_array <<< $(grep -v "#" \ - ${tfd}/hoststack_daily/${DUT}-${NODENESS}-${FLAVOR}.md | - awk {"$awk_nics_sub_cmd"} || echo "perftest") || die - SELECTION_MODE="--test" ;; *"ndrpdr-weekly"* ) - readarray -t test_tag_array <<< $(grep -v "#" \ - ${tfd}/ndrpdr_weekly/${DUT}-${NODENESS}-${FLAVOR}.md | - awk {"$awk_nics_sub_cmd"} || echo "perftest") || die - SELECTION_MODE="--test" ;; *"mrr-daily"* ) - readarray -t test_tag_array <<< $(grep -v "#" \ - ${tfd}/mrr_daily/${DUT}-${NODENESS}-${FLAVOR}.md | - awk {"$awk_nics_sub_cmd"} || echo "perftest") || die - SELECTION_MODE="--test" ;; *"mrr-weekly"* ) - readarray -t test_tag_array <<< $(grep -v "#" \ - ${tfd}/mrr_weekly/${DUT}-${NODENESS}-${FLAVOR}.md | - awk {"$awk_nics_sub_cmd"} || echo "perftest") || die - SELECTION_MODE="--test" ;; *"soak-weekly"* ) - readarray -t test_tag_array <<< $(grep -v "#" \ - ${tfd}/soak_weekly/${DUT}-${NODENESS}-${FLAVOR}.md | - awk {"$awk_nics_sub_cmd"} || echo "perftest") || die - SELECTION_MODE="--test" ;; *"report-iterative"* ) test_sets=(${TEST_TAG_STRING//:/ }) - # Run only one test set per run - report_file=${test_sets[0]}.md - readarray -t test_tag_array <<< $(grep -v "#" \ - ${tfd}/report_iterative/${NODENESS}-${FLAVOR}/${report_file} | - awk {"$awk_nics_sub_cmd"} || echo "perftest") || die - SELECTION_MODE="--test" ;; *"report-coverage"* ) test_sets=(${TEST_TAG_STRING//:/ }) - # Run only one test set per run - report_file=${test_sets[0]}.md - readarray -t test_tag_array <<< $(grep -v "#" \ - ${tfd}/report_coverage/${NODENESS}-${FLAVOR}/${report_file} | - awk {"$awk_nics_sub_cmd"} || echo "perftest") || die - SELECTION_MODE="--test" ;; * ) if [[ -z "${TEST_TAG_STRING-}" ]]; then # If nothing is specified, we will run pre-selected tests by # following tags. - test_tag_array=("mrrAND${default_nic}AND1cAND64bANDethip4-ip4base" - "mrrAND${default_nic}AND1cAND78bANDethip6-ip6base" - "mrrAND${default_nic}AND1cAND64bANDeth-l2bdbasemaclrn" - "mrrAND${default_nic}AND1cAND64bANDeth-l2xcbase" - "!drv_af_xdp" "!drv_avf") + test_tag_array=("mrrAND1cAND64bANDethip4-ip4base") else # If trigger contains tags, split them into array. test_tag_array=(${TEST_TAG_STRING//:/ }) fi - SELECTION_MODE="--include" + SELECTION_MODE="tags" ;; esac - # Blacklisting certain tags per topology. - # - # Reasons for blacklisting: - # - ipsechw - Blacklisted on testbeds without crypto hardware accelerator. - case "${TEST_CODE}" in - *"1n-vbox") - test_tag_array+=("!avf") - test_tag_array+=("!vhost") - test_tag_array+=("!flow") - ;; - *"1n-alt") - test_tag_array+=("!flow") - ;; - *"2n-icx") - test_tag_array+=("!ipsechw") - ;; - *"2n-spr") - ;; - *"2n-zn2") - test_tag_array+=("!ipsechw") - ;; - *"3n-alt") - test_tag_array+=("!ipsechw") - ;; - *"2n-grc") - test_tag_array+=("!ipsechw") - ;; - *"2n-emr") - ;; - *"3n-emr") - ;; - *"3n-oct") - ;; - *"3n-icx") - test_tag_array+=("!ipsechw") - test_tag_array+=("!3_node_double_link_topoANDnic_intel-xxv710") - ;; - *"3n-snr") - ;; - *"3n-icxd") - ;; - *"3na-spr") - ;; - *"3nb-spr") - ;; - *"1n-aws" | *"2n-aws" | *"3n-aws") - test_tag_array+=("!ipsechw") - ;; - *"2n-c7gn" | *"3n-c7gn") - test_tag_array+=("!ipsechw") - ;; - *"1n-c6in" | *"2n-c6in" | *"3n-c6in") - test_tag_array+=("!ipsechw") - ;; - *"-x-2n"* | *"-x-3n"*) - ;; - esac - - # We will add excluded NICs. - test_tag_array+=("${exclude_nics[@]/#/!NIC_}") - TAGS=() prefix="" - if [[ "${TEST_CODE}" != *"daily"* ]]; then - if [[ "${TEST_CODE}" == "vpp-"* ]]; then - if [[ "${TEST_CODE}" != *"device"* ]]; then - # Automatic prefixing for VPP perf jobs to limit the NIC used. - if [[ "${TEST_TAG_STRING-}" != *"nic_"* ]]; then - prefix="${default_nic}AND" - fi - fi - fi - fi set +x for tag in "${test_tag_array[@]}"; do if [[ "${tag}" == "!"* ]]; then @@ -1226,7 +1002,6 @@ function select_topology () { # - TOPOLOGIES_DIR - Path to existing directory with available topologies. # Variables set: # - TOPOLOGIES - Array of paths to suitable topology yaml files. - # - TOPOLOGIES_TAGS - Tag expression selecting tests for the topology. # Functions called: # - die - Print to stderr and exit. @@ -1235,104 +1010,78 @@ function select_topology () { case "${TEST_CODE}" in *"1n-aws") TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*1n-aws*.yaml ) - TOPOLOGIES_TAGS="1_node_single_link_topo" ;; *"1n-c6in") TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*1n-c6in*.yaml ) - TOPOLOGIES_TAGS="1_node_single_link_topo" ;; *"1n-alt" | *"1n-spr") TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*vpp_device*.template ) - TOPOLOGIES_TAGS="2_node_single_link_topo" ;; *"1n-vbox") TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*vpp_device*.template ) - TOPOLOGIES_TAGS="2_node_single_link_topo" ;; *"2n-aws") TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*2n-aws*.yaml ) - TOPOLOGIES_TAGS="2_node_single_link_topo" ;; *"2n-c7gn") TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*2n-c7gn*.yaml ) - TOPOLOGIES_TAGS="2_node_single_link_topo" ;; *"2n-c6in") TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*2n-c6in*.yaml ) - TOPOLOGIES_TAGS="2_node_single_link_topo" ;; *"2n-icx") TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*2n_icx_*.yaml ) - TOPOLOGIES_TAGS="2_node_*_link_topo" ;; *"2n-spr") TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*2n_spr_*.yaml ) - TOPOLOGIES_TAGS="2_node_*_link_topo" ;; *"2n-zn2") TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*2n_zn2_*.yaml ) - TOPOLOGIES_TAGS="2_node_*_link_topo" ;; *"3n-alt") TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3n_alt_*.yaml ) - TOPOLOGIES_TAGS="3_node_single_link_topo" ;; *"2n-grc") TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*2n_grc_*.yaml ) - TOPOLOGIES_TAGS="2_node_single_link_topo" ;; *"2n-emr") TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*2n_emr_*.yaml ) - TOPOLOGIES_TAGS="2_node_*_link_topo" ;; *"3n-emr") TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3n_emr_*.yaml ) - TOPOLOGIES_TAGS="3_node_*_link_topo" ;; *"3n-oct") TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3n_oct_*.yaml ) - TOPOLOGIES_TAGS="3_node_*_link_topo" ;; *"3n-aws") TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3n-aws*.yaml ) - TOPOLOGIES_TAGS="3_node_single_link_topo" ;; *"3n-c7gn") TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3n-c7gn*.yaml ) - TOPOLOGIES_TAGS="3_node_single_link_topo" ;; *"3n-c6in") TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3n-c6in*.yaml ) - TOPOLOGIES_TAGS="3_node_single_link_topo" ;; *"3n-icx") TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3n_icx_*.yaml ) - # Trailing underscore is needed to distinguish from 3n_icxd. - TOPOLOGIES_TAGS="3_node_*_link_topo" ;; *"3n-icxd") TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3n_icxd_*.yaml ) - TOPOLOGIES_TAGS="3_node_single_link_topo" ;; *"3n-snr") TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3n_snr_*.yaml ) - TOPOLOGIES_TAGS="3_node_single_link_topo" ;; *"3na-spr") TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3na_spr_*.yaml ) - TOPOLOGIES_TAGS="3_node_*_link_topo" ;; *"3nb-spr") TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*3nb_spr_*.yaml ) - TOPOLOGIES_TAGS="3_node_*_link_topo" ;; *"-x-2n"*) TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*_x_"${NODENESS}_${FLAVOR}"*.yaml ) - TOPOLOGIES_TAGS="2_node_single_link_topo" ;; *"-x-3n"*) TOPOLOGIES=( "${TOPOLOGIES_DIR}"/*_x_"${NODENESS}_${FLAVOR}"*.yaml ) - TOPOLOGIES_TAGS="3_node_single_link_topo" ;; *) # No falling back to default, that should have been done diff --git a/resources/libraries/python/suite_generator/constants.py b/resources/libraries/python/suite_generator/constants.py new file mode 100644 index 0000000000..fa729b76ec --- /dev/null +++ b/resources/libraries/python/suite_generator/constants.py @@ -0,0 +1,228 @@ +# Copyright (c) 2025 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Constants used by the Suite Generator. + +TODO: Consider reading environment variables to set some of the constants. +""" + + +import generator_functions as gf + + +# Logging +LOGGING_LEVEL = ("NOTSET", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL") +DEFAULT_LOG_LEVEL = "INFO" + +TEST_TYPES = ("mrr", "ndrpdr", "soak", "hoststack") + +# Directory with job specifications: +DIR_JOB_SPEC = "../../../job_specifications" +# The directory with test templates: +DIR_TESTS_IN = "../../../.." + +# Default path for generated json and md files. +DEFAULT_OUTPUT_PATH = "./out" +# Default filename for generated files. +DEFAULT_OUTPUT_FILE = "job_spec" + + +# Parameters of test(s) applied to the test groups. Their order in this tuple +# defines the order in the output MD file. +TEST_PARAMS = ("core", "framesize", "test-type", "infra") + +# Values used in the template files (NIC, driver, test type). +TMPL_NIC = "10ge2p1x710" +TMPL_DRV = "vfio-pci" +TMPL_TTYPE = "ndrpdr" + +# The line which separates test specifications from the rest of file. +TMPL_SEP_TESTS = "*** Test Cases ***\n" + +TMPL_TEST = { + "default": """ +| $frame_str-$cores_str-$driver$suite_id-$test_type +| | [Tags] | $frame_tag | $cores_tag +| | frame_size=$frame_num | phy_cores=$cores_num +""", + "trex": """ +| $frame_str--$suite_id-$test_type +| | [Tags] | $frame_tag +| | frame_size=$frame_num +""", + "iperf3": """ +| 128KB-$cores_str-$suite_id-mrr +| | [Tags] | 128KB | $cores_tag +| | frame_size=$frame_num | phy_cores=$cores_num +""", + "hoststack_cps_rps": """ +| $frame_str-$cores_str-$driver$suite_id +| | [Tags] | $frame_str | $cores_str +| | frame_size=$frame_num | phy_cores=$cores_num +""", + "hoststack_bps": """ +| $frame_str-$cores_str-$driver$suite_id +| | [Tags] | $cores_tag +| | phy_cores=$cores_num +""" +} + +# Mappings + +# Functions used to generate particular kind of suite +GEN_SUITE_PARAMS = { + "default": gf.generate_suite_params_default, + "trex": gf.generate_suite_params_trex, + "iperf3": gf.generate_suite_params_iperf3, + "hoststack_cps": gf.generate_suite_params_hoststack_cps_rps, + "hoststack_rps": gf.generate_suite_params_hoststack_cps_rps, + "hoststack_bps": gf.generate_suite_params_hoststack_bps +} +GEN_TEST = { + "default": gf.generate_test_default, + "trex": gf.generate_test_trex, + "iperf3": gf.generate_test_iperf3, + "hoststack_cps": gf.generate_test_hoststack_cps, + "hoststack_rps": gf.generate_test_hoststack_rps, + "hoststack_bps": gf.generate_test_hoststack_bps +} + +NIC_CODE_TO_NAME = { + "10ge2p1x710": "Intel-X710", + "40ge2p1xl710": "Intel-XL710", + "25ge2p1xxv710": "Intel-XXV710", + "25ge2p1e810xxv": "Intel-E810XXV", + "25ge2p1e822cq": "Intel-E822CQ", + "25ge2p1e823c": "Intel-E823C", + "100ge2p1e810cq": "Intel-E810CQ", + "50ge1p1ena": "Amazon-Nitro-50G", + "100ge1p1ena": "Amazon-Nitro-100G", + "200ge1p1ena": "Amazon-Nitro-200G", + "100ge2p1cx556a": "Mellanox-CX556A", + "100ge2p1cx6dx": "Mellanox-CX6DX", + "200ge2p1cx7veat": "Mellanox-CX7VEAT", + "200ge6p3cx7veat": "Mellanox-CX7VEAT", + "100ge2p1a063": "Cavium-A063-100G", + "1ge1p82540em": "virtual", +} + +# Tags to differentiate tests for different NIC driver. +NIC_DRIVER_TO_TAG = { + "vfio-pci": "DRV_VFIO_PCI", + "oct-vfio-pci": "DRV_VFIO_PCI", + "avf": "DRV_AVF", + "rdma-core": "DRV_RDMA_CORE", + "mlx5_core": "DRV_MLX5_CORE", + "af_xdp": "DRV_AF_XDP", + "tap": "DRV_TAP", + "vhost": "DRV_VHOST" +} + +# Suite names have to be different, add prefix. +NIC_DRIVER_TO_SUITE_PREFIX = { + "vfio-pci": "", + "oct-vfio-pci": "", + "avf": "avf", + "rdma-core": "rdma", + "mlx5_core": "mlx5", + "af_xdp": "af_xdp", + "tap": "", + "vhost": "" +} +# Driver name from job spec to driver variable mapping. +NIC_DRIVER_TO_VARIABLE = { + "vfio-pci": "vfio-pci", + "oct-vfio-pci": "vfio-pci", + "avf": "avf", + "rdma-core": "rdma-core", + "mlx5_core": "mlx5_core", + "af_xdp": "af_xdp" +} +DRIVERS_NOT_IN_NAME = ( + "vfio-pci", + "oct-vfio-pci", + "tap", + "vhost", + "-" +) + +# Each driver needs different plugin to work. +NIC_DRIVER_TO_PLUGINS = { + "vfio-pci": "dpdk_plugin.so", + "oct-vfio-pci": "dev_octeon_plugin.so", + "avf": "dev_iavf_plugin.so", + "rdma-core": "rdma_plugin.so", + "mlx5_core": "dpdk_plugin.so", + "af_xdp": "af_xdp_plugin.so" +} + +# Number of virtual functions of physical nic. +NIC_DRIVER_TO_VFS = { + "vfio-pci": "nic_vfs}= | 0", + "oct-vfio-pci": "nic_vfs}= | 0", + "avf": "nic_vfs}= | 1", + "rdma-core": "nic_vfs}= | 0", + "mlx5_core": "nic_vfs}= | 0", + "af_xdp": "nic_vfs}= | 0", + "tap": "nic_vfs}= | 0", + "vhost": "nic_vfs}= | 0" +} + +# Number of physical interfaces of physical nic. +NIC_CODE_TO_PFS = { + "10ge2p1x710": "nic_pfs}= | 2", + "40ge2p1xl710": "nic_pfs}= | 2", + "25ge2p1xxv710": "nic_pfs}= | 2", + "25ge2p1e810xxv": "nic_pfs}= | 2", + "25ge2p1e822cq": "nic_pfs}= | 2", + "25ge2p1e823c": "nic_pfs}= | 2", + "100ge2p1e810cq": "nic_pfs}= | 2", + "50ge1p1ena": "nic_pfs}= | 2", + "100ge1p1ena": "nic_pfs}= | 2", + "200ge1p1ena": "nic_pfs}= | 2", + "100ge2p1cx556a": "nic_pfs}= | 2", + "100ge2p1cx6dx": "nic_pfs}= | 2", + "200ge2p1cx7veat": "nic_pfs}= | 2", + "200ge6p3cx7veat": "nic_pfs}= | 6", + "100ge2p1a063": "nic_pfs}= | 2", + "1ge1p82540em": "nic_pfs}= | 2" +} + +PERF_TYPE_TO_KEYWORD = { + "mrr": "Traffic should pass with maximum rate", + "ndrpdr": "Find NDR and PDR intervals using optimized search", + "soak": "Find critical load using PLRsearch" +} + +PERF_TYPE_TO_SUITE_DOC_VER = { + "mrr": """fication:** In MaxReceivedRate tests TG sends traffic at \\ +| ... | line rate and reports total received packets over trial period. \\""", + "ndrpdr": """rification:** TG finds and reports throughput NDR (Non \\ +| ... | Drop Rate) with zero packet loss tolerance and throughput PDR \\ +| ... | (Partial Drop Rate) with non-zero packet loss tolerance (LT) \\ +| ... | expressed in percentage of packets transmitted. NDR and PDR are \\ +| ... | discovered for different Ethernet L2 frame sizes using MLRsearch \\ +| ... | library.""", + "soak": """rification:** TG sends traffic at dynamically computed \\ +| ... | rate as PLRsearch algorithm gathers data and improves its estimate \\ +| ... | of a rate at which a prescribed small fraction of packets \\ +| ... | would be lost. After set time, the serarch stops \\ +| ... | and the algorithm reports its current estimate. \\""" +} + +PERF_TYPE_TO_TEMPLATE_DOC_VER = { + "mrr": """Measure MaxReceivedRate for ${frame_size}B frames \\ +| | ... | using burst trials throughput test. \\""", + "ndrpdr": "Measure NDR and PDR values using MLRsearch algorithm.", + "soak": "Estimate critical rate using PLRsearch algorithm. \\" +} diff --git a/resources/libraries/python/suite_generator/generator.py b/resources/libraries/python/suite_generator/generator.py new file mode 100644 index 0000000000..d6ca242b6f --- /dev/null +++ b/resources/libraries/python/suite_generator/generator.py @@ -0,0 +1,272 @@ +# Copyright (c) 2025 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generate the test suites defined in the job specification. + +Only those defined in the job specification. +""" + + +import logging + +from glob import glob +from itertools import product +from os import path, makedirs, sep +from shutil import copyfile + +import constants as C + + +def _get_suite_type(tmpl_name: str) -> str: + """Determine the type of suite based on template name. + + :param tmpl_name: Template name + :type tmpl_name: str + :returns: Type of suite. + :rtype: str + """ + if tmpl_name.endswith("-tg-ndrpdr.robot"): + return "trex" + elif "hoststack" in tmpl_name: + if tmpl_name.endswith("-cps.robot"): + return "hoststack_cps" + elif tmpl_name.endswith("-rps.robot"): + return "hoststack_rps" + elif tmpl_name.endswith("-bps.robot"): + return "hoststack_bps" + else: + raise NotImplementedError( + f"Processing of {tmpl_name} is not implemented.") + elif tmpl_name.endswith("-iperf3-mrr.robot"): + # gso + return "iperf3" + elif tmpl_name.endswith("-ndrpdr.robot"): + # MUST be the last one. + # General NDRPDR, MRR and SOAK tests. + return "default" + else: + raise NotImplementedError( + f"Processing of {tmpl_name} is not implemented.") + + +def _find_test_tmpl(patern: str, test_tag: str) -> str: + """Find the right test template based on the test tag. + + To speed up the search process: + 1. Find the candidates based on the file name and only then + 2. look into the files and check the test tag. + + The test tag MUST be lowercase and it MUST be at the end of line, in the + best case, it is the only tag on the line. + + :param patern: The patern defining the path to the file used by glob. + :param: test_tag: The test tag as it is defined in the test template. + :type patern: str + :type test_tag: str + :returns: The path to the found test template. + :rtype: str + """ + # At first, find the candidates - the test name is in the file name... + for file_name in glob(patern, recursive=True): + if "/vpp/device/" in file_name: + # We do not use these tests anymore, but they are still in repo. + continue + with open(file_name, "rt") as fr: + # ... then check the tag inside the file. + if f"| {test_tag}\n" in fr.read(): + return file_name.removeprefix(C.DIR_TESTS_IN)[1:] + + +def _generate_suite(src: str, dst: str, test_tag: str, infra: tuple, + params: tuple) -> int: + """Generate a test sute. + + :param src: Path to the template file (suite). + :param dst: Path to the file where the generated suite is written. + :param test_tag: Test tag. + :param infra: NIC, driver. + :param params: core, framesize, test type. + :type src: str + :type dst: str + :type test_tag: str + :type infra: tuple + :type params: tuple + :returns: Return code: 0 - OK, 1 - Not OK + :rtype: int + """ + logging.debug( + f"Template file:\n{src}\n" + f"Destination file:\n{dst}\n" + f"Test name:\n{test_tag}\n" + f"Infra:\n{infra}\n" + f"Parameters:\n{params}\n" + ) + + tmpl_lines = list() + with open(src, "rt") as fr: + tmpl_lines = [line for line in fr] + if not tmpl_lines: + logging.error(f"The template file {src} is empty.") + return 1 + + # Extract the part with copyright, settings, variables and keywords + new_suite = "".join(tmpl_lines[:tmpl_lines.index(C.TMPL_SEP_TESTS) + 1]) + + # Determine the kind of suite: + try: + suite_type = _get_suite_type(src) + except NotImplementedError as err: + logging.error(err) + return 1 + + # Replace items in template + try: + new_suite = \ + C.GEN_SUITE_PARAMS[suite_type](new_suite, infra, params[0][1], dst) + except (ValueError, KeyError) as err: + logging.error(repr(err)) + return 1 + + # Add tests + try: + for param in params: + new_suite += C.GEN_TEST[suite_type](test_tag, infra, param) + except (TypeError, KeyError, ValueError) as err: + logging.error(repr(err)) + return 1 + + # Write the generated suite + try: + with open(dst, "wt") as fw: + fw.write(new_suite) + except IOError as err: + logging.error(f"Cannot write the file '{dst}'\n{err}") + return 1 + + return 0 + + +def generate_suites(output_dir: str, spec: dict, job: str) -> int: + """Generate all test suites defined in the specification. + + - job spec must include only one job + - iterate through tests (items "tests" in dicts in the list) + - find template robot file (suite) with tests (glob with patern=tag and + then look inside the file for the tag) + - replace parameters + - generate tests with cores/framesizes/nics/drivers/test-types + - The structure (dirs and files) must be the same as origin, incl __init__ + files. + - Robot framework runs ALL tests + + :param output_dir: Directory to write the generated suites to. + :param spec: Job specification. + :param job: Job which wil run the generated tests. + :type output_dir: str + :type spec: dict + :type job: str + :returns: Return code: 0 - OK, 1 - Not OK + :rtype: int + """ + + # Pre-process spec - generate combinations of params for each test + tests = dict() + for group in spec: + l_params = list() + l_nic_drv = list() + l_core_fsize = list() + for param in C.TEST_PARAMS: + if param not in group: + logging.error(f"The parameter '{param}' is mandatory.") + return 1 + if param == "core": + continue + elif param == "infra": + for nic, drv_lst in group[param].items(): + for drv in drv_lst: + l_nic_drv.append((nic, drv)) + elif param == "framesize": + for fsize in group[param]: + if isinstance(fsize, dict): + for fs, cores in fsize.items(): + for core in cores: + l_core_fsize.append((core, fs)) + else: # int, str + for core in group["core"]: + l_core_fsize.append((core, fsize)) + l_params.append(l_core_fsize) + elif isinstance(group[param], str): + l_params.append([group[param], ]) + else: # list + l_params.append(group[param]) + for test in group["tests"]: + if tests.get(test, None) is None: + tests[test] = dict() + for nic_drv in l_nic_drv: + tests[test][nic_drv] = list(product(*l_params)) + dut=job.split("-")[1] + if dut == "csit": + dut=job.split("-")[0] + if dut == "trex": # TRex runs on 2n testbeds, but the tests are 1n. + nodes = f"1n*-" + elif "-2n" in job: + nodes = f"2n*-" + elif "-1n" in job: + nodes = f"1n*-" + else: # e.g. 3n has no info about number of nodes + nodes = str() + for test_tag, infras in tests.items(): + logging.info(f"Generating testsuites for '{test_tag}' ...") + # Look for template files + patern = ( + f"{C.DIR_TESTS_IN}{sep}tests{sep}{dut}{sep}**{sep}{nodes}" + f"{C.TMPL_NIC}-{test_tag}*.robot" + ) + if "-ldpreload-nginx-" in test_tag: # hoststack nginx + # We need to generate both cps and rps + test_tag = test_tag.rsplit("-", 1)[0] + full_path = _find_test_tmpl(patern, test_tag) + if full_path is None: + logging.error("The template file not found.") + return 1 + file_path, file_name = path.split(full_path) + # Create the whole dir structure + out_path = path.join(output_dir, file_path) + makedirs(out_path, exist_ok=True) + # Process the file + for infra, suite_params in infras.items(): + # Create the file name: change NIC, add driver, change testtype + nic_drv = infra[0] # NIC + if infra[1] not in C.DRIVERS_NOT_IN_NAME: # driver + # if infra[1] not in ("vfio-pci", "tap", "vhost", "-"): # driver + nic_drv += f"-{C.NIC_DRIVER_TO_SUITE_PREFIX[infra[1]]}" + dst_name = file_name.replace(C.TMPL_NIC, nic_drv).\ + replace(C.TMPL_TTYPE, suite_params[0][1]) + # Generate the content of the suite + if _generate_suite(src=path.join(C.DIR_TESTS_IN, full_path), + dst=path.join(out_path, dst_name), test_tag=test_tag, + infra=infra, params=suite_params): + return 1 + + # Add __init__.robot files + tmp_path = str() + for dir in full_path.split(sep)[:-1]: + tmp_path = path.join(tmp_path, dir) + dst_path = path.join(output_dir, tmp_path, "__init__.robot") + if path.exists(dst_path): + continue # No need to do the same milion times + src_path = path.join(C.DIR_TESTS_IN, tmp_path, "__init__.robot") + if path.exists(src_path): # Is there an init file? + copyfile(src_path, dst_path) + + return 0 diff --git a/resources/libraries/python/suite_generator/generator_functions.py b/resources/libraries/python/suite_generator/generator_functions.py new file mode 100644 index 0000000000..27f9feb588 --- /dev/null +++ b/resources/libraries/python/suite_generator/generator_functions.py @@ -0,0 +1,491 @@ +# Copyright (c) 2025 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generate particular test suites. +""" + + +from string import Template + +import constants as C + + +def _replace_defensively(text: str, to_replace: str, replace_with: str, + how_many: int, in_filename: str, msg: str) -> str: + """Replace substrings while checking the number of occurrences. + + Return edited copy of the text. Assuming "whole" is really a string, + or something else with .replace not affecting it. + + :param text: The text to perform replacements on. + :param to_replace: Substring occurrences of which to replace. + :param replace_with: Substring to replace occurrences with. + :param how_many: Number of occurrences to expect. + :param msg: Error message to raise. + :param in_filename: File name in which the error occurred. + :type text: str + :type to_replace: str + :type replace_with: str + :type how_many: int + :type msg: str + :type in_filename: str + :returns: The whole text after replacements are done. + :rtype: str + :raises ValueError: If number of occurrences does not match. + """ + if text.count(to_replace) != how_many: + raise ValueError(f"{in_filename}: Parameter '{to_replace}': {msg}") + return text.replace(to_replace, replace_with) + + +def generate_suite_params_default(tmpl: str, infra: tuple, test_type: str, + file_name: str) -> str: + """Generate test parameters, documentation, tags, local template, ... + for the default (and the most common) test suite - pure ndrpdr, mrr. + + :param tmpl: Suite template. + :param infra: NIC, driver. + :param test_type: The test type - ndrpdr, mrr, ... + :param file_name: The file name for the test suite. It is used here only for + error messages. + :type tmpl: str + :type infra: tuple[str, str] + :type test_type: str + :type file_name: str + :returns: The common part of the suite. + :rtype: str + """ + + nic, driver = infra + + # Suite tags: test type, driver; the NIC is replaced later + tmpl = _replace_defensively( + tmpl, C.TMPL_TTYPE.upper(), test_type.upper(), 1, file_name, + "Suite type should appear once in uppercase (as a tag)." + ) + tmpl = _replace_defensively( + tmpl, C.NIC_DRIVER_TO_TAG[C.TMPL_DRV], C.NIC_DRIVER_TO_TAG[driver], 1, + file_name, "Driver tag should appear once." + ) + + # Documentation + tmpl = _replace_defensively( + tmpl, C.PERF_TYPE_TO_SUITE_DOC_VER[C.TMPL_TTYPE], + C.PERF_TYPE_TO_SUITE_DOC_VER[test_type], 1, file_name, + "Exact suite type documentation not found." + ) + + # Variables: NIC name, driver, plugin, NIC VFs + tmpl = _replace_defensively( + tmpl, C.NIC_CODE_TO_NAME[C.TMPL_NIC], C.NIC_CODE_TO_NAME[nic], 2, + file_name, "NIC name should appear twice (tag and variable)." + ) + tmpl = _replace_defensively( + tmpl, C.TMPL_DRV, C.NIC_DRIVER_TO_VARIABLE[driver], 1, file_name, + "Driver name should appear once." + ) + if "DPDK" not in tmpl: + tmpl = _replace_defensively( + tmpl, C.NIC_DRIVER_TO_PLUGINS[C.TMPL_DRV], + C.NIC_DRIVER_TO_PLUGINS[driver], 1, file_name, + "Driver plugin should appear once." + ) + tmpl = _replace_defensively( + tmpl, C.NIC_DRIVER_TO_VFS[C.TMPL_DRV], + C.NIC_DRIVER_TO_VFS[driver], 1, file_name, + "NIC VFs argument should appear once." + ) + tmpl = _replace_defensively( + tmpl, C.NIC_CODE_TO_PFS[C.TMPL_NIC], + C.NIC_CODE_TO_PFS[nic], 1, file_name, + "NIC PFs argument should appear once." + ) + + # Local (test) template: Documentation + tmpl = _replace_defensively( + tmpl, C.PERF_TYPE_TO_TEMPLATE_DOC_VER[C.TMPL_TTYPE], + C.PERF_TYPE_TO_TEMPLATE_DOC_VER[test_type], 1, file_name, + "Exact template type documentation not found." + ) + + # Local (test) template: Keyword performing the search + tmpl = _replace_defensively( + tmpl, C.PERF_TYPE_TO_KEYWORD[C.TMPL_TTYPE], + C.PERF_TYPE_TO_KEYWORD[test_type], 1, file_name, + "Main search keyword should appear once in the suite." + ) + + return tmpl + + +def generate_suite_params_trex(tmpl: str, infra: tuple, test_type: str, + file_name: str) -> str: + """Generate test parameters, documentation, tags, local template, ... + for the TRex test suite - pure ndrpdr, mrr. + + :param tmpl: Suite template. + :param infra: NIC, driver. + :param test_type: The test type - ndrpdr, mrr, ... + :param file_name: The file name for the test suite. It is used here only for + error messages. + :type tmpl: str + :type infra: tuple[str, str] + :type test_type: str + :type file_name: str + :returns: The common part of the suite. + :rtype: str + """ + + nic = infra[0] + + # Suite tags: test type; the NIC is replaced later + tmpl = _replace_defensively( + tmpl, C.TMPL_TTYPE.upper(), test_type.upper(), 1, file_name, + "Suite type should appear once in uppercase (as tag)." + ) + + # Documentation + tmpl = _replace_defensively( + tmpl, C.PERF_TYPE_TO_SUITE_DOC_VER["ndrpdr"], + C.PERF_TYPE_TO_SUITE_DOC_VER[test_type], 1, file_name, + "Exact suite type doc not found." + ) + + # Variables: NIC name, plugin, NIC VFs + tmpl = _replace_defensively( + tmpl, C.NIC_CODE_TO_NAME[C.TMPL_NIC], C.NIC_CODE_TO_NAME[nic], 2, + file_name, "NIC name should appear twice (tag and variable)." + ) + + # Local (test) template: Documentation + tmpl = _replace_defensively( + tmpl, C.PERF_TYPE_TO_TEMPLATE_DOC_VER[C.TMPL_TTYPE], + C.PERF_TYPE_TO_TEMPLATE_DOC_VER[test_type], 1, file_name, + "Exact template type documentation not found." + ) + + # Local (test) template: Keyword performing the search + tmpl = _replace_defensively( + tmpl, C.PERF_TYPE_TO_KEYWORD[C.TMPL_TTYPE], + C.PERF_TYPE_TO_KEYWORD[test_type], 1, file_name, + "Main search keyword should appear once in the suite." + ) + + return tmpl + + +def generate_suite_params_iperf3(tmpl: str, infra: tuple, test_type: str, + file_name: str) -> str: + """Generate test parameters, documentation, tags, local template, ... + for the GSO (iperf3) test suite. + + :param tmpl: Suite template. + :param infra: NIC, driver. + :param test_type: The test type - ndrpdr, mrr, ... + :param file_name: The file name for the test suite. It is used here only for + error messages. + :type tmpl: str + :type infra: tuple[str, str] + :type test_type: str + :type file_name: str + :returns: The common part of the suite. + :rtype: str + """ + + nic = infra[0] + _ = test_type + + # Variables: NIC name, plugin, NIC VFs + tmpl = _replace_defensively( + tmpl, C.NIC_CODE_TO_NAME[C.TMPL_NIC], C.NIC_CODE_TO_NAME[nic], 2, + file_name, "NIC name should appear twice (tag and variable)." + ) + + return tmpl + + +def generate_suite_params_hoststack_cps_rps(tmpl: str, infra: tuple, + test_type: str, file_name: str) -> str: + """Generate test parameters, documentation, tags, local template, ... + for the hoststack cps and rps test suite. + + :param tmpl: Suite template. + :param infra: NIC, driver. + :param test_type: The test type - ndrpdr, mrr, ... + :param file_name: The file name for the test suite. It is used here only for + error messages. + :type tmpl: str + :type infra: tuple[str, str] + :type test_type: str + :type file_name: str + :returns: The common part of the suite. + :rtype: str + """ + + nic, driver = infra + _ = test_type + + # Variables: NIC name, plugin, NIC VFs + tmpl = _replace_defensively( + tmpl, C.NIC_CODE_TO_NAME[C.TMPL_NIC], C.NIC_CODE_TO_NAME[nic], 2, + file_name, "NIC name should appear twice (tag and variable)." + ) + tmpl = _replace_defensively( + tmpl, C.NIC_DRIVER_TO_TAG[C.TMPL_DRV], C.NIC_DRIVER_TO_TAG[driver], 1, + file_name, "Driver tag should appear once." + ) + tmpl = _replace_defensively( + tmpl, C.TMPL_DRV, C.NIC_DRIVER_TO_VARIABLE[driver], 1, file_name, + "Driver name should appear once." + ) + tmpl = _replace_defensively( + tmpl, C.NIC_DRIVER_TO_PLUGINS[C.TMPL_DRV], + C.NIC_DRIVER_TO_PLUGINS[driver], 1, file_name, + "Driver plugin should appear once." + ) + tmpl = _replace_defensively( + tmpl, C.NIC_DRIVER_TO_VFS[C.TMPL_DRV], + C.NIC_DRIVER_TO_VFS[driver], 1, file_name, + "NIC VFs argument should appear once." + ) + tmpl = _replace_defensively( + tmpl, C.NIC_CODE_TO_PFS[C.TMPL_NIC], + C.NIC_CODE_TO_PFS[nic], 1, file_name, + "NIC PFs argument should appear once." + ) + + return tmpl + + +def generate_suite_params_hoststack_bps(tmpl: str, infra: tuple, + test_type: str, file_name: str) -> str: + """Generate test parameters, documentation, tags, local template, ... + for the hoststack bps test suite. + + :param tmpl: Suite template. + :param infra: NIC, driver. + :param test_type: The test type - ndrpdr, mrr, ... + :param file_name: The file name for the test suite. It is used here only for + error messages. + :type tmpl: str + :type infra: tuple[str, str] + :type test_type: str + :type file_name: str + :returns: The common part of the suite. + :rtype: str + """ + + nic, driver = infra + _ = test_type + + # Variables: NIC name, plugin, NIC VFs + tmpl = _replace_defensively( + tmpl, C.NIC_CODE_TO_NAME[C.TMPL_NIC], C.NIC_CODE_TO_NAME[nic], 2, + file_name, "NIC name should appear twice (tag and variable)." + ) + tmpl = _replace_defensively( + tmpl, C.NIC_DRIVER_TO_TAG[C.TMPL_DRV], C.NIC_DRIVER_TO_TAG[driver], 1, + file_name, "Driver tag should appear once." + ) + tmpl = _replace_defensively( + tmpl, C.TMPL_DRV, C.NIC_DRIVER_TO_VARIABLE[driver], 1, file_name, + "Driver name should appear once." + ) + tmpl = _replace_defensively( + tmpl, C.NIC_DRIVER_TO_PLUGINS[C.TMPL_DRV], + C.NIC_DRIVER_TO_PLUGINS[driver], 1, file_name, + "Driver plugin should appear once." + ) + tmpl = _replace_defensively( + tmpl, C.NIC_DRIVER_TO_VFS[C.TMPL_DRV], + C.NIC_DRIVER_TO_VFS[driver], 1, file_name, + "NIC VFs argument should appear once." + ) + tmpl = _replace_defensively( + tmpl, C.NIC_CODE_TO_PFS[C.TMPL_NIC], + C.NIC_CODE_TO_PFS[nic], 1, file_name, + "NIC PFs argument should appear once." + ) + + return tmpl + + +def generate_test_default(test_tag: str, infra: tuple, params: tuple) -> str: + """Generate test (only one) for the default (and the most common) test + suite - pure ndrpdr, mrr. + + :param test_tag: Test tag. + :param infra: NIC, driver. + :param params: The test parameters - core, framesize, test type. + :type test_tag: str + :type infra: tuple[str, str] + :type params: tuple + :returns: The test. + :rtype: str + """ + + test = Template(C.TMPL_TEST["default"]) + (cores, frame), test_type = params + mapping = { + "suite_id": test_tag, + "test_type": test_type, + "driver": "" if infra[1] in C.DRIVERS_NOT_IN_NAME else \ + f"{C.NIC_DRIVER_TO_SUITE_PREFIX[infra[1]]}-", + "frame_tag": "IMIX" if frame == "imix" else f"{frame}B", + "frame_str": "IMIX" if frame == "imix" else f"{frame}B", + "frame_num": "IMIX_v4_1" if frame == "imix" else f"${{{frame}}}", + "cores_tag": f"{cores}C", + "cores_str": f"{cores}c", + "cores_num": f"${{{cores:d}}}" + } + return test.safe_substitute(mapping) + + +def generate_test_trex(test_tag: str, infra: tuple, params: tuple) -> str: + """Generate test (only one) for the TRex test suite - pure ndrpdr, mrr. + + :param test_tag: Test tag. + :param infra: NIC, driver. + :param params: The test parameters - core, framesize, test type. + :type test_tag: str + :type infra: tuple[str, str] + :type params: tuple + :returns: The test. + :rtype: str + """ + + test = Template(C.TMPL_TEST["trex"]) + (_, frame), test_type = params + _ = infra + mapping = { + "suite_id": test_tag, + "test_type": test_type, + "frame_tag": "IMIX" if frame == "imix" else f"{frame}B", + "frame_str": "IMIX" if frame == "imix" else f"{frame}B", + "frame_num": "IMIX_v4_1" if frame == "imix" else f"${{{frame}}}" + } + return test.safe_substitute(mapping) + + +def generate_test_iperf3(test_tag: str, infra: tuple, params: tuple) -> str: + """Generate test (only one) for the GSO (iperf3) test suite. + + :param test_tag: Test tag. + :param infra: NIC, driver. + :param params: The test parameters - core, framesize, test type. + :type test_tag: str + :type infra: tuple[str, str] + :type params: tuple + :returns: The test. + :rtype: str + """ + + test = Template(C.TMPL_TEST["iperf3"]) + (cores, frame), _ = params + _ = infra + mapping = { + "suite_id": test_tag, + "frame_num": "IMIX_v4_1" if frame == "imix" else f"${{{frame}}}", + "cores_tag": f"{cores}C", + "cores_str": f"{cores}c", + "cores_num": f"${{{cores:d}}}" + } + return test.safe_substitute(mapping) + + +def generate_test_hoststack_cps(test_tag: str, infra: tuple, + params: tuple) -> str: + """Generate test (only one) for the hoststack cps and rps test suite. + + :param test_tag: Test tag. + :param infra: NIC, driver. + :param params: The test parameters - core, framesize, test type. + :type test_tag: str + :type infra: tuple[str, str] + :type params: tuple + :returns: The test. + :rtype: str + """ + + test = Template(C.TMPL_TEST["hoststack_cps_rps"]) + (cores, frame), _ = params + mapping = { + "suite_id": f"{test_tag}-cps", + "driver": "" if infra[1] in C.DRIVERS_NOT_IN_NAME else \ + f"{C.NIC_DRIVER_TO_SUITE_PREFIX[infra[1]]}-", + "frame_tag": "IMIX" if frame == "imix" else f"{frame}B", + "frame_str": "IMIX" if frame == "imix" else f"{frame}B", + "frame_num": "IMIX_v4_1" if frame == "imix" else f"${{{frame}}}", + "cores_tag": f"{cores}C", + "cores_str": f"{cores}c", + "cores_num": f"${{{cores:d}}}" + } + return test.safe_substitute(mapping) + + +def generate_test_hoststack_rps(test_tag: str, infra: tuple, + params: tuple) -> str: + """Generate test (only one) for the hoststack cps and rps test suite. + + :param test_tag: Test tag. + :param infra: NIC, driver. + :param params: The test parameters - core, framesize, test type. + :type test_tag: str + :type infra: tuple[str, str] + :type params: tuple + :returns: The test. + :rtype: str + """ + + test = Template(C.TMPL_TEST["hoststack_cps_rps"]) + (cores, frame), _ = params + mapping = { + "suite_id": f"{test_tag}-rps", + "driver": "" if infra[1] in C.DRIVERS_NOT_IN_NAME else \ + f"{C.NIC_DRIVER_TO_SUITE_PREFIX[infra[1]]}-", + "frame_tag": "IMIX" if frame == "imix" else f"{frame}B", + "frame_str": "IMIX" if frame == "imix" else f"{frame}B", + "frame_num": "IMIX_v4_1" if frame == "imix" else f"${{{frame}}}", + "cores_tag": f"{cores}C", + "cores_str": f"{cores}c", + "cores_num": f"${{{cores:d}}}" + } + return test.safe_substitute(mapping) + + +def generate_test_hoststack_bps(test_tag: str, infra: tuple, + params: tuple) -> str: + """Generate test (only one) for the hoststack bps test suite. + + :param test_tag: Test tag. + :param infra: NIC, driver. + :param params: The test parameters - core, framesize, test type. + :type test_tag: str + :type infra: tuple[str, str] + :type params: tuple + :returns: The test. + :rtype: str + """ + + test = Template(C.TMPL_TEST["hoststack_bps"]) + (cores, frame), _ = params + mapping = { + "suite_id": test_tag, + "driver": "" if infra[1] in C.DRIVERS_NOT_IN_NAME else \ + f"{C.NIC_DRIVER_TO_SUITE_PREFIX[infra[1]]}-", + "frame_str": "IMIX" if frame == "imix" else f"{frame}B", + "cores_tag": f"{cores}C", + "cores_str": f"{cores}c", + "cores_num": f"${{{cores:d}}}" + } + return test.safe_substitute(mapping) diff --git a/resources/libraries/python/suite_generator/spec_processor.py b/resources/libraries/python/suite_generator/spec_processor.py new file mode 100644 index 0000000000..1542959081 --- /dev/null +++ b/resources/libraries/python/suite_generator/spec_processor.py @@ -0,0 +1,252 @@ +# Copyright (c) 2025 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Process the specification provided as a YAML file. +""" + + +import logging + +from copy import deepcopy +from itertools import product +from os import path +from yaml import load, FullLoader, YAMLError + +import constants as C + + +def _get_job_params(in_str: str) -> list: + """Get the parameters from the name of job. + + The function searches for parameters enclosed by '{' and '}'. + Typicaly 'node-arch', ... + + :params in_str: Input string + :type in_str: str + :returns: A list of parameters found in the input string. + :rtype: list + """ + params = list() + idx_end = 0 + while True: + idx = in_str.find("{", idx_end) + if idx == -1: + break + idx_end = in_str.find("}", idx) + if idx_end == -1: + break + params.append(in_str[idx+1:idx_end]) + return params + + +def process_specification() -> dict: + """Process the specification provided as a YAML file. + + :param path_to_spec: Path to YAML file with specification. + :returns: Full specification with all parameters replaced by their values. + :rtype: dict + """ + + raw_spec = dict() + for spec_file in ("jobs.yaml", "test_sets.yaml", "test_groups.yaml"): + path_to_spec = path.join(C.DIR_JOB_SPEC, spec_file) + try: + with open(path_to_spec, "r") as file_read: + spec_part = load(file_read, Loader=FullLoader) + except IOError as err: + logging.error( + f"Not possible to open the file {path_to_spec}\n" + f"{err}" + ) + return dict() + except YAMLError as err: + logging.error( + f"An error occurred while parsing the specification file " + f"{path_to_spec}\n{err}" + ) + return dict() + + raw_spec.update(spec_part) + + jobs = raw_spec.get("jobs", dict()) + + # Expand the "jobs" section, do not add test sets + spec = dict() + for job_name in jobs: + job_params = _get_job_params(job_name) + job_tmpl = jobs[job_name] + + if not job_params: + spec[job_name] = deepcopy(job_tmpl) + else: + l_opts = list() + top_params = list() + for param in job_params: + try: + job_tmpl_param = job_tmpl[param] + top_params.append(param) + except KeyError: + continue + if isinstance(job_tmpl_param, str): + l_opts.append([job_tmpl_param, ]) + elif isinstance(job_tmpl_param, list): + if param == "node-arch": + l_itms = list() + for itm in job_tmpl_param: + if isinstance(itm, str): + l_itms.append(itm) + elif isinstance(itm, dict): + l_itms.append(list(itm.keys())[0]) + else: + logging.error( + f"{job_name}: " + "Not allowed data type in 'node-arch'." + ) + return dict() + l_opts.append(l_itms) + else: + l_opts.append(job_tmpl_param) + elif isinstance(job_tmpl_param, dict): + l_opts.append(list(job_tmpl_param.keys())) + + opt_combs = product(*l_opts) + for opt_comb in opt_combs: + d_params = dict(zip(top_params, opt_comb)) + for param in job_params: + if param in d_params: + continue + for top_param in top_params: + top_val = job_tmpl[top_param] + if isinstance(top_val, dict): + try: + val = top_val[d_params[top_param]].\ + get(param, None) + except AttributeError: + logging.error( + f"{job_name}: The parameter '{param}' must " + f"be defined." + ) + return dict() + if val is not None: + d_params[param] = val + continue + try: + new_job_name = job_name.format(**d_params) + except KeyError as err: + logging.error( + f"{job_name}: One or more parameters is not defined: " + f"{err}." + ) + return dict() + spec[new_job_name] = deepcopy(job_tmpl) + for top_param in top_params: + # Delete top parameters, they are not needed anymore: + if top_param not in C.TEST_PARAMS: + del spec[new_job_name][top_param] + + # Get tests, they must be in "node-arch" branch: + if top_param == "node-arch": + for itm in job_tmpl["node-arch"]: + if isinstance(itm, str): + if itm == d_params["node-arch"]: + spec[new_job_name]["test-set"] = str() + else: # dict + key = list(itm.keys())[0] + if key == d_params["node-arch"]: + spec[new_job_name]["test-set"] = itm[key] + return { + "jobs": spec, + "test-sets": raw_spec.get("test-sets", dict()), + "test-groups": raw_spec.get("test-groups", dict()) + } + + +def generate_job_spec(spec: dict, job: str, test_set: str, + test_type: str) -> dict: + """Generate a full specification for the required job. + + :param spec: Job specification extracted from the specification file. + :param job: Job name. + :param test_set: Test set to be used for the full job specification. + :param test_type: Test type. + :type spec: dict + :type job: str + :type test_set: str + :type test_type: str + :returns: Full job specification. + :rtype: dict + """ + + try: + job_spec = spec["jobs"][job] + except KeyError as err: + logging.error(f"Job {job} not in specification.\n{err}") + return dict() + try: + test_sets = spec["test-sets"] + except KeyError as err: + logging.error(f"No test sets defined.\n{err}") + return dict() + try: + test_groups = spec["test-groups"] + except KeyError as err: + logging.error(f"No test groups defined.\n{err}") + return dict() + + # If the test set is provided from cmd line, replace that in the + # specification, or add it if there is no test set specified. + if test_set: + job_spec["test-set"] = test_set + + # If the test type is provided from cmd line, replace that in the + # specification, or add it if there is no test set specified. + if test_type: + job_spec["test-type"] = test_type + + # Take the required job and add "test-set" and "test-group" sub-structures: + try: + job_spec["tests"] = deepcopy(test_sets[job_spec["test-set"]]) + new_tests = list() + for tests in job_spec["tests"]["tests"]: + new_test = dict() + if isinstance(tests, str): + new_test = {"tests": test_groups[tests]} + new_test["group-name"] = tests + elif isinstance(tests, dict): + for key in tests: + new_test["tests"] = test_groups[key] + new_test["group-name"] = key + new_test.update(tests[key]) + else: + return dict() + new_tests.append(new_test) + job_spec["tests"]["tests"] = deepcopy(new_tests) + except KeyError as err: + logging.error(f"{job}: One or more parameters is not defined: {err}.") + return dict() + + # Add all parameters to test groups: + for group in job_spec["tests"]["tests"]: + params = dict() + for param in job_spec: + if param not in C.TEST_PARAMS: + continue + params[param] = job_spec[param] + for param in job_spec["tests"]: + if param not in C.TEST_PARAMS: + continue + params[param] = job_spec["tests"][param] + params.update(group) + group.update(params) + + return job_spec["tests"]["tests"] diff --git a/resources/libraries/python/suite_generator/suite_generator.py b/resources/libraries/python/suite_generator/suite_generator.py new file mode 100755 index 0000000000..5ebf29b721 --- /dev/null +++ b/resources/libraries/python/suite_generator/suite_generator.py @@ -0,0 +1,858 @@ +#!/usr/bin/python3 + +# Copyright (c) 2025 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +=============== +Suite Generator +=============== + +The suite generator generates the test suites defined in the specification files +and chosen by command-line parameters. + +Input +===== + +The mandatory input information is: +1. Job name (string). The test suites are generated for this job. Only one job + can be specified. +2. Directory to store generated tests. + +The optional or mandatory input information, depending on the job type, is: +3. Trigger parameters (string). Mandatory for on-demand jobs (iterative, + coverage, bisect, ...). This string includes information directly from the + gerrit trigger. To distinguish between information for the Robot (e.g. tags), + the information for the suite generator starts with the hash tag #. E.g.: + csit-vpp-report-iter-2n-aws-perftest #2n-aws-vpp-iterative #ndrpdr + If we include all neccessary information to this string, we do not need to + use "Test set" and "Test type" parameters described below. +4. Test set (string). Mandatory for on-demand jobs (iterative, coverage, bisect, + ...). Specifies the tests to run. Only one test set can be specified. + If the test set is not specified as command-line parameter, it must be + specified in the specification file. +5. Test type (string). Mandatory for some of on-demand jobs. Only one test type + from this list can be specified: + - mrr, + - ndrpdr, + - hoststack + - soak. + +The optional input information is (if it is not provided, the values specified +in constants.py are used): +6. Output directory (string). Directory to store generated files. If not given, + the default directory is used. +7. Output file name (string). The name of output file: + - transformed and expanded yaml file to json. + If not given, the default name is used. +8. Create JSON file (boolean). If set, the processed specification will be + writen to the output directory as a JSON file. The default value is "False". +9. Logging level (string). One of "NOTSET", "DEBUG", "INFO", "WARNING", "ERROR", + "CRITICAL". The default value is "INFO". + +Output +====== + +The output is: +1. A directory structure with generated tests, suites and other necessary files + to run tests using Robot Framework. +2. Optionaly the JSON file with expaned specification. + +The return code is: +0 - if everything is OK, +1 - if anything went wrong. + +Job specification +================= + +The jobs are specified in the specification YAML files. There are three files +specifying: +1. test groups (test_groups.yaml) +2. test sets (test_sets.yaml) +3. jobs (jobs.yaml) +The default location is the directory `resources/job_specifications/`. + +Test groups +----------- + +The test group is a named list of tests defined by their test tag. It is the +elementary part of the job specification. + +Size of a test group: + ++-------------+--------------+---------+ +| Name | Abbreviation | Size | ++=============+==============+=========+ +| small | sm | 1 - 2 | +| medium | md | 3 - 4 | +| large | lg | 5 - 8 | +| extra large | xl | 9 - 16 | +| 2x large | xxl | 17 - 32 | ++-------------+--------------+---------+ + +It is not recommended to use 2xl test groups. + +Example: + + trex-ip4-sm: + - ethip4-ip4base-tg + - ethip4-ip4scale20k-tg + trex-nat44-cps-md: + - ethip4tcp-ip4base-h1024-p63-s64512-cps-tg + - ethip4tcp-ip4base-h262144-p63-s16515072-cps-tg + - ethip4tcp-ip4base-h1024-p63-s64512-cps-tg + - ethip4tcp-ip4base-h262144-p63-s16515072-cps-tg + +Test sets +--------- + +A test set consists of: +- test parameters (infra, core, framesize) and +- test groups, optionally with test parameters. + +The test parameters are: +- number of cores (list), +- framesize (list), +- infrastructure - NIC and driver (dictionary). + +The test parameters can be defined for the whole test set and/or for each test +group separately. + +Examples: + +1. Definition of parameters + + 2n-aws-vpp-iterative: + core: [1, 2] + framesize: [64, 1518] + infra: + nitro-50g: + - vfio-pci + tests: + - ip4-sm + - ip6-sm: + framesize: [78, 1518] + - l2-md + +In this example, parameters defined outside `tests` (core, framesize, infra) +are valid for all test groups listed in `tests`, but parameters defined directly +for a test group (ip6-sm in this example) are valid only for this test group and +overwrite parameters defined for the whole test set. + +2. Parameters defined outside of test sets + + 2n-spr-dpdk-iterative: + tests: + - dpdk-small + +In this example, no parameters are defined, so parameters defined in the section +`jobs` are used. + +3. Driver not defined + + 2n-spr-trex-iterative: + infra: + 100ge2p1e810cq: + - "-" + 200ge2p1cx7veat: + - "-" + tests: + - trex-ip4-sm + - trex-nat44-cps-md + - trex-nat44-tput-md: + framesize: [100, ] + - trex-ip6-sm: + framesize: [78, ] + - trex-l2-sm + +In this example, no drivers ("-") are defined as these tests do not need them to +specify. + +4. Cores not defined + + csit-trex-perf-report-coverage-{stream}-{node-arch}: + stream: + - "2506" + test-type: ndrpdr + framesize: [64, 1518, 9000, "imix"] + core: ["-", ] + node-arch: + - 2n-icx: 2n-icx-trex-coverage + - 2n-spr: 2n-spr-trex-coverage + +In this example, no cores ("-") are defined as these tests do not need them to +specify. + +5. Extended "framesize" parameter + + 2n-emr-vpp-cov-ip6-00: + infra: + 100ge2p1e810cq: + - avf + - vfio-pci + framesize: + - 78 + - 1518: [1, 2] + - 9000: [1, ] + - imix + tests: + - ip6-acl-md + +If we need to run limited or extended number of cores for particular framesize, +we can specify them as a list for chosen framesize. In this example: + +framesize: + - 78 Cores defined in the job will be applied. + - 1518: [1, 2] Only [1, 2] cores will be aplied. + - 9000: [1, ] Only [1, ] core will be aplied. + - imix Cores defined in the job will be applied. + +For more examples, see the specification yaml files. + +Jobs +---- + +A job consists of: +- job parameters + - test-type (string), + - stream (list), + - node-arch (list), + - ... any other. +- global test parameters, see above. + +A special parameter is the `node-arch` whitch assignes a test set to the +testbed. It is a list of testbeds defined for the job. The items can be strings +or dictionaries: + +node-arch: + - 2n-icx, + - 2n-spr: 2n-spr-vpp-iterative + + +- The testbed `2n-icx` is defined for this job but without a test set. The test + set MUST be specified as a command-line parameter. This approach is typical + for iterative, coverage, verify, etc jobs which can run more then one test + set. +- The testbed `2n-spr` is defined for this job with a test set. This test set + CAN be changed by a command-line parameter. This approach is used with daily + and weekly jobs as they run always the same test set. + +Examples: + +1. daily, weekly + + csit-vpp-perf-mrr-daily-master-{node-arch}: + test-type: mrr + framesize: + - 64b + core: [1, 2, 4] + node-arch: + - 2n-icx: 2n-icx-vpp-iterative + - 2n-spr: 2n-spr-vpp-iterative + - 2n-zn2: 2n-zn2-vpp-iterative + - 3n-alt: 3n-alt-vpp-iterative + - 3n-icx: 3n-icx-vpp-iterative + - 3n-icxd: 3n-icxd-vpp-iterative + - 3n-snr: 3n-snr-vpp-iterative + - 3na-spr: 3na-spr-vpp-iterative + - 3nb-spr: 3nb-spr-vpp-iterative + +In this example, we defined 9 jobs, one for each item in the `node-arch` list. + +There are four parameters to completly define a test. One of them, `test-type`, +is always common for all tests, so it is defined in the job specification. +The rest, `core`, `framesize` and `infra` can vary from test to test, but all +must be defined, no matter on which level (job, test set). + +2. iterative + + csit-vpp-perf-report-iterative-{stream}-{node-arch}: + stream: + - "2502" + - "2410" + framesize: + - 64 + core: [1, 2, 4] + node-arch: + - 2n-aws: 2n-aws-vpp-iterative + - 2n-c6in: 2n-c6in-vpp-iterative + - 2n-c7gn: 2n-c7gn-vpp-iterative + - 2n-icx + - 2n-spr + - 2n-zn2 + - 3n-alt + - 3n-icx + - 3n-icxd + - 3n-snr + - 3na-spr + - 3nb-spr: 3nb-spr-vpp-iterative + +In this example, we defined 24 jobs as each `node-arch` has two `stream`s. +Four items in `node-arch` list have assigned their test sets (they can be +re-defined by command-line argument) and the rest must be assigned by +command-line argument. + +Processing of the specification +------------------------------- + +There are three levels of specification: +1. jobs - top level +2. tests-sets +3. test-groups - bottom level + +The specification is processed from top to down and only for required job +specified by the command line parameter. +The test set specified in the job is replaced by its definition, and the test +groups in it are replaced by their definitions. Then we set all parameters +(`test-type`, `core`, `framesize` and `infra`) for each test, again, from top to +down, overwriting top values by bottom values. So, if e.g. framesize is +specified on all levels, the lowest one is used. + +Triggering +========== + +The Suite generator is launched from Jenkins / Github Actions either as a +periodical job (daily, weekly, ...) or on-demand job (iterative, coverage, +verify, bisect, ...). + +Periodical jobs +--------------- + +Main characteristics: +- no trigger, +- detected "daily" or "weekly" in the job name, +- job name e.g.: csit-dpdk-perf-mrr-weekly-master-2n-grc. + +Parameters: +- job-name - mandatory. + +Robot Framework parameters (relevant to Suite generator): +- directory with generated tests +- runs all tests + +Example: + +1. Periodical jobs + +./suite_generator.py --gen-tests-dir generated \ + --job csit-trex-perf-ndrpdr-weekly-master-2n-spr +./suite_generator.py --gen-tests-dir generated \ + --job csit-vpp-perf-mrr-daily-master-2n-icx + +Generates tests for the job "csit-trex-perf-ndrpdr-weekly-master-2n-spr" and +stores them in "generated" directory. +As these jobs are periodical jobs, no more parameters are needed. + +On-demand jobs +-------------- + +Main characteristics: +- started by a gerrit trigger, examples of currently used triggers: + - jobs for the report: + - csit-vpp-report-iter-2n-aws-perftest vpp-mrr-00 + - csit-dpdk-report-iter-2n-icx-perftest dpdk-ndrpdr-00 + - csit-vpp-report-cov-2n-aws-perftest vpp-00 + - csit-vpp-report-cov-2n-icx-perftest ip4-05 + - verify jobs: + - csit-2n-zn2-perftest + - bisect jobs: + - bisecttest-2n-spr d35f7f098 +- job name, e.g.: + - report: + - csit-dpdk-perf-report-coverage-2410-2n-icx + - verify: + - csit-vpp-perf-verify-master-2n-zn2 + - bisect: + - vpp-csit-bisect-master-ubuntu2404-x86_64-2n-spr + +Parameters: +- job-name - mandatory +- test-set - optional + - Mandatory for iterative and coverage jobs. + - By default all verify and bisect jobs use "iterative" test set + - If needed, one of "coverage" test sets can be specified as command line + parameter. +- test-type - optional + - Default: mrr + - If needed, test type can be specified as command line parameter. + +Robot Framework parameters (relevant to Suite generator): +- directory with tests +- set of tags for verify and bisect jobs + +Examples: + +1. + +./suite_generator.py --gen-tests-dir generated \ + --job csit-vpp-perf-report-iterative-2506-2n-icx +./suite_generator.py --gen-tests-dir generated \ + --job csit-vpp-perf-report-iterative-2506-2n-icx --test-type ndrpdr +./suite_generator.py --gen-tests-dir generated \ + --job csit-vpp-perf-report-coverage-2506-2n-spr \ + --test-set 2n-spr-vpp-cov-ip4-00 + + +User guide +========== + +.. or how to run the tests + +The trigger consist of + - trigger name + - tags (optional) + - trigger parameters (optional) - each parameter starts with the hash tag # + and parameters are separated by the white space. + +Periodical jobs +--------------- + +The user does not use these jobs as they are periodicaly triggered by Jenkins. +The only parameter they get, is the name of Jenkins job. + +On-demand jobs +-------------- + +Report +...... + +Both iterative and coverage jobs are triggered by user from gerrit. + +Iterative jobs +,,,,,,,,,,,,,, + +Trigger name: + - csit-vpp-report-iter---perftest + +Default values: + - Iterative test set - the same as it is used for periodical jobs + - Test type: mrr + +Mandatory trigger parameters: + - No mandatory parameters - default values are used in that case + +Optional trigger parameters: + - test set + - test type + +Examples: + +1. csit-vpp-report-iter-2n-emr-perftest + + - Generates default test set and default test type. + +2. csit-vpp-report-iter-2n-emr-perftest #ndrpdr + + - Generates default test set and ndr pdr tests + +3. csit-vpp-report-iter-2n-emr-perftest #2n-emr-vpp-hoststack + csit-vpp-report-iter-2n-emr-perftest #2n-emr-vpp-hoststack #hoststack + + - Generates hoststack test set. + - It is not necessary to specify the test type as it is defined in the name of + the test set, so both examples are equal and correct. + +Coverage jobs +,,,,,,,,,,,,, + +Trigger name: + - csit-vpp-report-cov---perftest + +Default values: + - Test type: ndrpdr + +Mandatory trigger parameters: + - Test set + +Optional trigger parameters: + - Test type, default value: ndrpdr + +Examples: + +1. csit-vpp-report-cov-2n-emr-perftest #2n-emr-vpp-cov-ip4-00 + + - Generates the defined test set, ndrpdr tests. + +Development +........... + +Verify jobs +,,,,,,,,,,, + +Trigger name: + - csit---perftest + +Default values: + - Iterative test set - the same as it is used for periodical jobs + - Test type: mrr + +Mandatory trigger parameters: + - No mandatory parameters - default values are used in that case + +Optional trigger parameters: + - test set + - test type + +Examples: + +1. csit-2n-emr-perftest + + - Generates iterative mrr tests, runs only those selected by tags. + +2. csit-2n-emr-perftest #2n-emr-vpp-cov-ip4-00 #ndrpdr + + - Generates defined test set, ndrpdr tests, runs only those selected by tags. + +Bisect jobs +,,,,,,,,,,, + +Trigger name: + - bisecttest-- + +Default values: + - Iterative test set - the same as it is used for periodical jobs + - Test type: mrr + +Mandatory trigger parameters: + - No mandatory parameters - default values are used in that case + +Optional trigger parameters: + - test set + - test type + +Examples: + +1. bisecttest-2n-emr + + - Generates iterative mrr tests, runs only those selected by tags. + +2. bisecttest-2n-emr #2n-emr-vpp-cov-ip4-00 #ndrpdr + + - Generates defined test set, ndrpdr tests, runs only those selected by tags. + +Special cases +,,,,,,,,,,,,, + +Tox +;;; + +If the job name includes the string "tox", all tests defined in all test sets +are generated. Tox job does not run any tests. + +generate-all +;;;;;;;;;;;; + +If the string "generate-all" is in the trigger, all tests defined in all test +sets are generated. + +This can be used with verify or bisect jobs if you do not know which test set +includes required test(s). Be careful and use proper tags to specify the +test(s). + +Examples: + +1. csit-2n-emr-perftest #generate-all + + - generates all tests specified in all test sets, runs only those selected by + tags. + +""" + + +import logging +import sys + +from argparse import ArgumentParser, RawTextHelpFormatter, BooleanOptionalAction +from json import dumps +from os import path, makedirs + +import constants as C + +from generator import generate_suites +from spec_processor import process_specification, generate_job_spec + + +def _get_job_type(job: str) -> str: + """Get job type based on its name. + + :param job: The job name. + :type job: str + :returns: The job type. + :rtype: str + :raises ValueError: If unknown job provided. + """ + + if "daily" in job: + return "periodical" + elif "weekly" in job: + return "periodical" + elif "iterative" in job: + return "iterative" + elif "coverage" in job: + return "coverage" + elif "tox" in job: + return "tox" + elif "verify" in job: + return "verify" + elif "bisect" in job: + return "bisect" + else: + raise ValueError(f"Unknown job type: {job}.") + + +def suite_generator(args) -> int: + """Suite generator + + Top level function. + + :param args: Parsed CLI arguments. + :type args: Namespace + :returns: Return code: 0 - OK, 1 - Not OK. + :rtype: int + """ + + # Process the comand line arguments + job = args.job.lower() + test_dir = args.gen_tests_dir + trigger_params = args.trigger_params + test_set = args.test_set.lower() + test_type = args.test_type.lower() + output_dir = args.output_dir if args.output_dir else C.DEFAULT_OUTPUT_PATH + output_file = \ + args.output_file if args.output_file else C.DEFAULT_OUTPUT_FILE + create_json = args.create_json + logging_level = \ + args.logging_level if args.logging_level else C.DEFAULT_LOG_LEVEL + + # Set the logging + logging.basicConfig( + format='%(asctime)s %(levelname)s: %(message)s', + datefmt="%Y-%m-%d %H:%M:%S", + level=logging_level + ) + + tr_prms = f"{trigger_params if trigger_params else 'Not specified'}\n" + logging.info( + "\nCommand line parameters:\n" + f"Job: {job}\n" + f"Test output dir: {test_dir}\n" + f"Trigger params: {tr_prms}" + f"Test set: {test_set if test_set else 'Not specified'}\n" + f"Test type: {test_type if test_type else 'Not specified'}\n" + f"Output dir: {output_dir}\n" + f"Output file: {output_file}\n" + f"Logging level: {logging_level}\n" + f"Create JSON: {create_json}\n" + ) + + try: + makedirs(test_dir, exist_ok=False) + logging.debug(f"Directory '{test_dir}' created successfully.") + except FileExistsError: + logging.info(f"The directory '{test_dir}' exists, it will be reused.") + except OSError as err: + logging.critical( + f"The directory '{test_dir}' cannot be created.\n{err}" + ) + return 1 + + try: + makedirs(output_dir, exist_ok=False) + logging.debug(f"Directory '{output_dir}' created successfully.") + except FileExistsError: + logging.info( + f"The directory '{output_dir}' exists, it will be reused " + f"but not emptied. Existing files will be overwritten, new files " + f"will be added." + ) + except OSError as err: + logging.critical( + f"The directory '{output_dir}' cannot be created.\n{err}" + ) + return 1 + + # Generate the job specification as a JSON structure: + spec = process_specification() + if not spec: + logging.critical( + f"The specification in {C.DIR_JOB_SPEC} cannot be processed." + ) + return 1 + + # Get the job type + try: + job_type = _get_job_type(job) + except ValueError as err: + logging.critical(err) + return 1 + + if job_type == "tox": + generate_all = True + + # Parse the "test tag" and get: + # - test type + # - test set + # If this information is not present, the values from cmd line or job + # specification will be used. + + generate_all = True if "generate-all" in trigger_params or \ + job_type == "tox" else False + if job_type in ("iterative", "coverage", "verify", "bisect"): + search_test_type, search_test_set = True, True + for tr_part in trigger_params.replace("#", "").split(" "): + if search_test_set: + if tr_part in spec["test-sets"]: + test_set = tr_part + search_test_set = False + if search_test_type: + for ttype in C.TEST_TYPES: + if ttype in tr_part: + test_type = ttype + search_test_type = False + + logging.info( + "\nInput parameters (after processing):\n" + f"Job: {job}\n" + f"Job type: {job_type}\n" + f"Test output dir: {test_dir}\n" + f"Trigger params: {trigger_params}\n" + f"Test set: {test_set}\n" + f"Test type: {test_type}\n" + f"Output dir: {output_dir}\n" + f"Output file: {output_file}\n" + f"Logging level: {logging_level}\n" + f"Create JSON: {create_json}\n\n" + ) + + # Write the specification to a JSON file: + if create_json: + try: + with open(path.join(output_dir, f"{output_file}.json"), "wt") as fw: + fw.write(dumps(spec, sort_keys=True, indent=2)) + except IOError as err: + logging.critical(f"Cannot write the JSON file.\n {err}") + return 1 + + # Generate suites: + if generate_all: + ret_val = 0 + for job in spec["jobs"]: + job_type = _get_job_type(job) + if job_type == "periodical": + logging.info(job) + if spec["jobs"][job]["test-type"] in ("hoststack", "soak"): + job_spec = generate_job_spec(spec, job, test_set, test_type) + if not job_spec: + return 1 + ret_val += generate_suites(test_dir, job_spec, job) + else: + for ttype in ("mrr", "ndrpdr"): + job_spec = generate_job_spec(spec, job, test_set, ttype) + if not job_spec: + return 1 + ret_val += generate_suites(test_dir, job_spec, job) + elif job_type == "coverage" and "vpp" in job: + logging.info(job) + tbed = "-".join(job.split("-")[-2:]) + "-" + for tset in spec["test-sets"]: + if "-cov-" in tset and tbed in tset: + for ttype in ("mrr", "ndrpdr"): + job_spec = generate_job_spec(spec, job, tset, ttype) + if not job_spec: + return 1 + ret_val += generate_suites(test_dir, job_spec, job) + return ret_val + else: + job_spec = generate_job_spec(spec, job, test_set, test_type) + if not job_spec: + return 1 + return generate_suites(test_dir, job_spec, job) + + +def parse_args(): + """Parse the command line arguments. + + :returns: Parsed command line arguments. + :rtype: Namespace + """ + parser = ArgumentParser( + formatter_class=RawTextHelpFormatter, + description=__doc__ + ) + parser.add_argument( + "--job", required=True, type=str, + help="The name of the job which will run the generated tests." + ) + parser.add_argument( + "--gen-tests-dir", required=True, type=str, + help=("Directory to store generated tests.") + ) + parser.add_argument( + "--trigger-params", required=False, type=str, default=str(), + help=("The parameters from trigger which triggered this build.") + ) + parser.add_argument( + "--test-set", required=False, type=str, default=str(), + help=( + "Test set (on-demand jobs (iterative, coverage, bisect, ...)).\n" + "Specifies the tests to run. Only one test set can be specified.\n" + "If the test set is not specified as command-line parameter, it\n" + "must be specified in the specification file." + ) + ) + parser.add_argument( + "--test-type", required=False, type=str, default=str(), + help=( + "Optional. If not specified, the test type from specification is\n" + "used.\n" + "Mandatory for on-demand (iterative, coverage, bisect, ...) jobs." + ) + ) + parser.add_argument( + "--output-dir", required=False, type=str, default=str(), + help=( + "Optional. Directory to store generated files. If not given, the\n" + "default directory is used." + ) + ) + parser.add_argument( + "--output-file", required=False, type=str, default=str(), + help=( + "Optional. The name of output file(s):\n" + "1. transformed and expanded yaml file to json, and\n" + "2. generated job specifications (if requested) in text format.\n" + " The file extension is 'md'.\n" + "If not given, the default name is used." + ) + ) + parser.add_argument( + "--create-json", required=False, type=bool, default=False, + action=BooleanOptionalAction, + help=( + "Optional. If set, the processed specification will be writen to\n" + "the output directory as a JSON file." + ) + ) + parser.add_argument( + "--logging-level", required=False, type=str, + choices=[i for i in C.LOGGING_LEVEL], + default=C.DEFAULT_LOG_LEVEL, + help=f"Optional. Logging level. (default: {C.DEFAULT_LOG_LEVEL})" + ) + return parser.parse_args() + + +if __name__ == "__main__": + """Entry point if called from cli. + """ + sys.exit(suite_generator(parse_args())) diff --git a/tox.ini b/tox.ini index c61677477f..cd1983d716 100644 --- a/tox.ini +++ b/tox.ini @@ -1,4 +1,4 @@ -# Copyright (c) 2024 Cisco and/or its affiliates. +# Copyright (c) 2025 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -44,6 +44,7 @@ script_dir = ./resources/libraries/bash/entry/tox [testenv:autogen] setenv = PYTHONPATH = {toxinidir} +passenv = JOB_NAME allowlist_externals = bash commands = bash {[tox]script_dir}/autogen.sh