From 8485f3bf773ad3b6ec4426dd2465b356b41c5a06 Mon Sep 17 00:00:00 2001 From: Peter Mikus Date: Thu, 30 Jun 2022 08:17:31 +0000 Subject: [PATCH] feat(infra): Add SNR Signed-off-by: Peter Mikus Change-Id: I4e39a59a192682c4bd9ebc7fa81ac727b55cf8e4 --- docs/lab/testbed_specifications.md | 60 +++++++++------- .../lf_inventory/host_vars/10.30.51.89.yaml | 33 +++++++++ .../lf_inventory/host_vars/10.30.51.93.yaml | 33 +++++++++ .../lf_inventory/host_vars/10.30.51.94.yaml | 33 +++++++++ fdio.infra.ansible/inventories/lf_inventory/hosts | 3 + fdio.infra.pxe/docker-dnsmasq/etc/dnsmasq.conf | 3 + .../tftpboot/grub/grub.cfg-a4:bf:01:55:ac:39 | 4 ++ .../tftpboot/grub/grub.cfg-a4:bf:01:55:af:5e | 4 ++ .../ubuntu_focal_amd64/a4:bf:01:55:ac:39/meta-data | 0 .../ubuntu_focal_amd64/a4:bf:01:55:ac:39/user-data | 83 ++++++++++++++++++++++ .../ubuntu_focal_amd64/a4:bf:01:55:af:5e/meta-data | 0 .../ubuntu_focal_amd64/a4:bf:01:55:af:5e/user-data | 83 ++++++++++++++++++++++ 12 files changed, 312 insertions(+), 27 deletions(-) create mode 100644 fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.89.yaml create mode 100644 fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.93.yaml create mode 100644 fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.94.yaml create mode 100644 fdio.infra.pxe/docker-dnsmasq/tftpboot/grub/grub.cfg-a4:bf:01:55:ac:39 create mode 100644 fdio.infra.pxe/docker-dnsmasq/tftpboot/grub/grub.cfg-a4:bf:01:55:af:5e create mode 100644 fdio.infra.pxe/docker-nginx/html/ubuntu_focal_amd64/a4:bf:01:55:ac:39/meta-data create mode 100644 fdio.infra.pxe/docker-nginx/html/ubuntu_focal_amd64/a4:bf:01:55:ac:39/user-data create mode 100644 fdio.infra.pxe/docker-nginx/html/ubuntu_focal_amd64/a4:bf:01:55:af:5e/meta-data create mode 100644 fdio.infra.pxe/docker-nginx/html/ubuntu_focal_amd64/a4:bf:01:55:af:5e/user-data diff --git a/docs/lab/testbed_specifications.md b/docs/lab/testbed_specifications.md index 9306177011..b7b794ee7c 100644 --- a/docs/lab/testbed_specifications.md +++ b/docs/lab/testbed_specifications.md @@ -934,14 +934,14 @@ FD.io CSIT lab contains following server types: - IPMI and host management ports. - NIC ports connected into 3-node testbed topology. - Main HW configuration: - - Chassis: ?. - - Motherboard: ?. - - Processors: ?* Intel Atom P5362B 2.2 GHz. - - RAM Memory: ?* ?GB DDR4-2933. + - Chassis: Intel JACOBSVILLE SDP. + - Motherboard: Intel JACOBSVILLE E63448-400. + - Processors: 1* Intel Atom P5362B 2.2 GHz. + - RAM Memory: 2* 16GB DDR4-2933. - Disks: ?* ? SATA SSD. - NICs configuration: - Numa0: (x16, PCIe3.0 lane) - - PCIe Slot yy:00.xx: ?. + - PCIe BuiltIn ec:00.xx: e810-XXVDA4-4p25GE Intel. ``` ### Naming Convention @@ -1851,24 +1851,30 @@ Note: Server19 (TG) is shared between testbed33 & testbed211 - IPMI IP: 10.30.50.93 - Host IP: 10.30.51.93 - portnames: - - s93-t39-sut1-c1/p1 - 100GE-port1 e810-DA2-2p100GE. - - s93-t39-sut1-c1/p2 - 100GE-port2 e810-DA2-2p100GE. + - s93-t39-sut1-c1/p1 - 25GE-port1 e810-XXVDA4-4p25GE. + - s93-t39-sut1-c1/p2 - 25GE-port2 e810-XXVDA4-4p25GE. + - s93-t39-sut1-c1/p2 - 25GE-port3 e810-XXVDA4-4p25GE. + - s93-t39-sut1-c1/p2 - 25GE-port4 e810-XXVDA4-4p25GE. - ServerG1 [Server-Type-G1]: - testbedname: testbed39. - hostname: s94-t39-sut2. - IPMI IP: 10.30.50.94 - Host IP: 10.30.51.94 - portnames: - - s94-t39-sut2-c1/p1 - 100GE-port1 e810-DA2-2p100GE. - - s94-t39-sut2-c1/p2 - 100GE-port1 e810-DA2-2p100GE. + - s94-t39-sut2-c1/p1 - 25GE-port1 e810-XXVDA4-4p25GE. + - s94-t39-sut2-c1/p2 - 25GE-port2 e810-XXVDA4-4p25GE. + - s94-t39-sut2-c1/p3 - 25GE-port3 e810-XXVDA4-4p25GE. + - s94-t39-sut2-c1/p4 - 25GE-port4 e810-XXVDA4-4p25GE. - ServerF4 [Server-Type-F4]: - testbedname: testbed39. - hostname: s89-t39t310-tg1. - IPMI IP: 10.30.50.89 - Host IP: 10.30.51.89 - portnames: - - s89-t39t310-tg1-c6/p1 - 100GE-port1 e810-DA2-2p100GE. - - s89-t39t310-tg1-c6/p2 - 100GE-port1 e810-DA2-2p100GE. + - s89-t39t310-tg1-c6/p1 - 25GE-port1 e810-XXVDA4-4p25GE. + - s89-t39t310-tg1-c6/p2 - 25GE-port2 e810-XXVDA4-4p25GE. + - s89-t39t310-tg1-c6/p3 - 25GE-port3 e810-XXVDA4-4p25GE. + - s89-t39t310-tg1-c6/p4 - 25GE-port4 e810-XXVDA4-4p25GE. ``` ### Per Testbed Wiring @@ -2292,10 +2298,13 @@ To be completed. ``` - testbed39: - - ring1 100GE-ports e810-DA2-2p100GE on SUTs and TG: + - ring1 25GE-ports e810-XXVDA4-4p25GE on SUTs and TG: - s89-t39t310-tg1-c6/p1 to s93-t39-sut1-c1/p1. - s93-t39-sut1-c1/p2 to s94-t39-sut2-c1/p2. - s94-t39-sut2-c1/p1 to s89-t39t310-tg1-c6/p2. + - s89-t39t310-tg1-c6/p3 to s93-t39-sut1-c1/p3. + - s93-t39-sut1-c1/p4 to s94-t39-sut2-c1/p4. + - s94-t39-sut2-c1/p3 to s89-t39t310-tg1-c6/p4. ``` ## Inventory @@ -2372,15 +2381,13 @@ To be completed. ``` 1. 10GE NICs - - 14 * Intel® Ethernet Converged Network Adapter X710-DA4 - - 6 * Intel® Ethernet Converged Network Adapter X710-DA2 - - 6 * Intel® Ethernet Converged Network Adapter X520-DA2 + - ? 2. 25GE NICs - - 12 * Intel® Ethernet Network Adapter XXV710-DA2 + - ? 3. 40GE NICs - - 2 * Intel® Ethernet Converged Network Adapter XL710-QDA2 + - ? 4. 100GE NICs - - 4 * mcx556a-edat NICs (not on site yet, in transit) + - ? ``` ### Pluggables and Cables @@ -2389,27 +2396,26 @@ Pluggables: ``` 1. 10GE SFP+ - - 16 * Intel E10GSFPSR Ethernet SFP+ SR Optics - - 80 * 10G SR optic (generic, "Intel" compatible branded) + - ? 2. 25GE SFP28 - - None + - ? 3. 40GE QSFP+ - - None + - ? 4. 100GE - - 8 * mcp1600-c002 qsfp28 pluggables and cables (not on site yet, in transit) + - ? ``` Standalone cables: ``` 1. 10GE - - None + - ? 2. 25GE - - None + - ? 3. 40GE QSFP+ - - 20 * Intel XLDACBL5 40G QSFP+ Passive DAC Cable + - ? 4. 100GE - - None + - ? ``` ### Other Parts diff --git a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.89.yaml b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.89.yaml new file mode 100644 index 0000000000..ba44916ab0 --- /dev/null +++ b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.89.yaml @@ -0,0 +1,33 @@ +--- +# file: host_vars/10.30.51.89.yaml + +hostname: "s89-t39t310-tg1" +grub: + audit: "0" + default_hugepagesz: "2M hugepagesz=1G hugepages=32 hugepagesz=2M hugepages=32768" + hpet: "disable" + intel_idle.max_cstate: "1" + intel_iommu: "on" + intel_pstate: "disable" + iommu: "pt" + isolcpus: "1-31,33-63,65-95,97-127" + mce: "off" + nmi_watchdog: "0" + nohz_full: "1-31,33-63,65-95,97-127" + nosoftlockup: true + numa_balancing: "disable" + processor.max_cstate: "1" + rcu_nocbs: "1-31,33-63,65-95,97-127" + tsc: "reliable" +sysctl: + kernel: + watchdog_cpumask: "0,32,64,96" + vm: + nr_hugepages: 32768 + max_map_count: 20000 + +inventory_ipmi_hostname: "10.30.50.89" +cpu_microarchitecture: "icelake" + +intel_700_matrix: "dpdk22.03" +intel_800_matrix: "dpdk22.03" diff --git a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.93.yaml b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.93.yaml new file mode 100644 index 0000000000..147d4c606f --- /dev/null +++ b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.93.yaml @@ -0,0 +1,33 @@ +--- +# file: host_vars/10.30.51.93.yaml + +hostname: "s93-t39-sut1" +grub: + audit: "0" + default_hugepagesz: "2M hugepagesz=1G hugepages=2 hugepagesz=2M hugepages=4096" + hpet: "disable" + intel_idle.max_cstate: "1" + intel_iommu: "on" + intel_pstate: "disable" + iommu: "pt" + isolcpus: "1-23" + mce: "off" + nmi_watchdog: "0" + nohz_full: "1-23" + nosoftlockup: true + numa_balancing: "disable" + processor.max_cstate: "1" + rcu_nocbs: "1-23" + tsc: "reliable" +sysctl: + kernel: + watchdog_cpumask: "0" + vm: + nr_hugepages: 4096 + max_map_count: 20000 + +inventory_ipmi_hostname: "10.30.50.93" +cpu_microarchitecture: "snowridge" + +intel_700_matrix: "dpdk21.02" +intel_800_matrix: "dpdk21.02" diff --git a/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.94.yaml b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.94.yaml new file mode 100644 index 0000000000..368ef7c689 --- /dev/null +++ b/fdio.infra.ansible/inventories/lf_inventory/host_vars/10.30.51.94.yaml @@ -0,0 +1,33 @@ +--- +# file: host_vars/10.30.51.94.yaml + +hostname: "s94-t39-sut2" +grub: + audit: "0" + default_hugepagesz: "2M hugepagesz=1G hugepages=2 hugepagesz=2M hugepages=4096" + hpet: "disable" + intel_idle.max_cstate: "1" + intel_iommu: "on" + intel_pstate: "disable" + iommu: "pt" + isolcpus: "1-23" + mce: "off" + nmi_watchdog: "0" + nohz_full: "1-23" + nosoftlockup: true + numa_balancing: "disable" + processor.max_cstate: "1" + rcu_nocbs: "1-23" + tsc: "reliable" +sysctl: + kernel: + watchdog_cpumask: "0" + vm: + nr_hugepages: 4096 + max_map_count: 20000 + +inventory_ipmi_hostname: "10.30.50.94" +cpu_microarchitecture: "snowridge" + +intel_700_matrix: "dpdk21.02" +intel_800_matrix: "dpdk21.02" diff --git a/fdio.infra.ansible/inventories/lf_inventory/hosts b/fdio.infra.ansible/inventories/lf_inventory/hosts index 413a8fb740..4a6b819e15 100644 --- a/fdio.infra.ansible/inventories/lf_inventory/hosts +++ b/fdio.infra.ansible/inventories/lf_inventory/hosts @@ -15,6 +15,7 @@ all: 10.30.51.84: #s84-t213-tg1 - icelake 10.30.51.86: #s86-t214-tg1 - icelake 10.30.51.88: #s88-t215-tg1 - icelake + 10.30.51.89: #s89-t39t310-tg1 - icelake 10.32.8.10: #s28-t26t35-tg1 - skylake 10.32.8.19: #s34-t27-tg1 - cascadelake 10.32.8.21: #s36-t28-tg1 - cascadelake @@ -41,6 +42,8 @@ all: 10.30.51.83: #s83-t213-sut1 - icelake 10.30.51.85: #s85-t214-sut1 - icelake 10.30.51.87: #s87-t215-sut1 - icelake + 10.30.51.93: #s93-t39-sut1 - snowridge + 10.30.51.94: #s94-t39-sut2 - snowridge 10.32.8.11: #s29-t26-sut1 - denverton 10.32.8.12: #s30-t35-sut1 - denverton 10.32.8.13: #s31-t35-sut2 - denverton diff --git a/fdio.infra.pxe/docker-dnsmasq/etc/dnsmasq.conf b/fdio.infra.pxe/docker-dnsmasq/etc/dnsmasq.conf index 3b812c397d..9ba1afd855 100755 --- a/fdio.infra.pxe/docker-dnsmasq/etc/dnsmasq.conf +++ b/fdio.infra.pxe/docker-dnsmasq/etc/dnsmasq.conf @@ -90,8 +90,11 @@ dhcp-host=3c:ec:ef:b9:a6:92,10.30.51.85,s85-t214-sut1,infinite # eno1 dhcp-host=3c:ec:ef:bc:92:e4,10.30.51.86,s86-t214-tg1,infinite # eno1 dhcp-host=3c:ec:ef:bc:92:e8,10.30.51.87,s87-t215-sut1,infinite # eno1 dhcp-host=3c:ec:ef:bc:a1:ac,10.30.51.88,s88-t215-tg1,infinite # eno1 +dhcp-host=?,10.30.51.89,s89-t39t310-tg1,infinite # eno1 dhcp-host=d8:5e:d3:02:b1:bb,10.30.51.91,s58-nomad,infinite #eno1 dhcp-host=d8:5e:d3:02:b1:9f,10.30.51.92,s59-nomad,infinite #eno1 +dhcp-host=a4:bf:01:55:ac:39,10.30.51.93,s93-t39-sut1,infinite # enp4s0 +dhcp-host=a4:bf:01:55:af:5e,10.30.51.94,s94-t39-sut2,infinite # enp4s0 # Hosts DHCP settings yul1_2 subnet. dhcp-host=ac:1f:6b:8b:54:c8,10.32.8.10,s28-t26t35-tg1,infinite # eno1 diff --git a/fdio.infra.pxe/docker-dnsmasq/tftpboot/grub/grub.cfg-a4:bf:01:55:ac:39 b/fdio.infra.pxe/docker-dnsmasq/tftpboot/grub/grub.cfg-a4:bf:01:55:ac:39 new file mode 100644 index 0000000000..279749f37b --- /dev/null +++ b/fdio.infra.pxe/docker-dnsmasq/tftpboot/grub/grub.cfg-a4:bf:01:55:ac:39 @@ -0,0 +1,4 @@ +menuentry "Install s93-t39-sut1" { + linux /casper/ubuntu-focal-amd64/vmlinuz ip=dhcp interface=a4:bf:01:55:ac:39 url=https://releases.ubuntu.com/20.04.4/ubuntu-20.04.4-live-server-amd64.iso autoinstall ds=nocloud-net\;s=http://10.30.51.28:8081/ubuntu_focal_amd64/a4:bf:01:55:ac:39/ --- console=ttyS0,115200n8 quiet + initrd /casper/ubuntu-focal-amd64/initrd +} diff --git a/fdio.infra.pxe/docker-dnsmasq/tftpboot/grub/grub.cfg-a4:bf:01:55:af:5e b/fdio.infra.pxe/docker-dnsmasq/tftpboot/grub/grub.cfg-a4:bf:01:55:af:5e new file mode 100644 index 0000000000..68e988c45f --- /dev/null +++ b/fdio.infra.pxe/docker-dnsmasq/tftpboot/grub/grub.cfg-a4:bf:01:55:af:5e @@ -0,0 +1,4 @@ +menuentry "Install s94-t39-sut2" { + linux /casper/ubuntu-focal-amd64/vmlinuz ip=dhcp interface=a4:bf:01:55:af:5e url=https://releases.ubuntu.com/20.04.4/ubuntu-20.04.4-live-server-amd64.iso autoinstall ds=nocloud-net\;s=http://10.30.51.28:8081/ubuntu_focal_amd64/a4:bf:01:55:af:5e/ --- console=ttyS0,115200n8 quiet + initrd /casper/ubuntu-focal-amd64/initrd +} diff --git a/fdio.infra.pxe/docker-nginx/html/ubuntu_focal_amd64/a4:bf:01:55:ac:39/meta-data b/fdio.infra.pxe/docker-nginx/html/ubuntu_focal_amd64/a4:bf:01:55:ac:39/meta-data new file mode 100644 index 0000000000..e69de29bb2 diff --git a/fdio.infra.pxe/docker-nginx/html/ubuntu_focal_amd64/a4:bf:01:55:ac:39/user-data b/fdio.infra.pxe/docker-nginx/html/ubuntu_focal_amd64/a4:bf:01:55:ac:39/user-data new file mode 100644 index 0000000000..45cf743fed --- /dev/null +++ b/fdio.infra.pxe/docker-nginx/html/ubuntu_focal_amd64/a4:bf:01:55:ac:39/user-data @@ -0,0 +1,83 @@ +#cloud-config +autoinstall: + version: 1 + apt: + geoip: true + preserve_sources_list: false + primary: + - arches: [amd64, i386] + uri: http://ca.archive.ubuntu.com/ubuntu + - arches: [default] + uri: http://ports.ubuntu.com/ubuntu-ports + identity: + hostname: s93-t39-sut1 + password: $6$Y62lhMGJD8YNzmJn$H4DSqjrwFp5WN3tOvIrF.g/G2duOV76zXHAmaA/RU8jfT8H8sDumLQe/Q.EmI5pjPv7dzgI8j9BQPWes7efBK0 + realname: testuser + username: testuser + keyboard: + layout: us + toggle: null + variant: '' + locale: en_US.UTF-8 + network: + ethernets: + eno1: + addresses: [ 10.30.51.93/24 ] + dhcp4: false + gateway4: 10.30.51.1 + nameservers: + addresses: [ 1.1.1.1, 8.8.8.8 ] + version: 2 + packages: + - python3 + refresh-installer: + update: no + ssh: + allow-pw: true + authorized-keys: [] + install-server: true + storage: + config: + - grub_device: false + id: disk-sda + path: /dev/sda + ptable: gpt + preserve: false + type: disk + wipe: superblock-recursive + - device: disk-sda + grub_device: true + id: partition-0 + flag: boot + number: 1 + preserve: false + size: 536870912 + type: partition + wipe: superblock + - id: format-0 + fstype: fat32 + preserve: false + type: format + volume: partition-0 + - device: disk-sda + flag: '' + grub_device: false + id: partition-1 + number: 2 + preserve: false + size: -1 + type: partition + wipe: superblock + - fstype: ext4 + id: format-1 + preserve: false + type: format + volume: partition-1 + - device: format-1 + id: mount-1 + path: / + type: mount + - device: format-0 + id: mount-0 + path: /boot/efi + type: mount \ No newline at end of file diff --git a/fdio.infra.pxe/docker-nginx/html/ubuntu_focal_amd64/a4:bf:01:55:af:5e/meta-data b/fdio.infra.pxe/docker-nginx/html/ubuntu_focal_amd64/a4:bf:01:55:af:5e/meta-data new file mode 100644 index 0000000000..e69de29bb2 diff --git a/fdio.infra.pxe/docker-nginx/html/ubuntu_focal_amd64/a4:bf:01:55:af:5e/user-data b/fdio.infra.pxe/docker-nginx/html/ubuntu_focal_amd64/a4:bf:01:55:af:5e/user-data new file mode 100644 index 0000000000..5d33054455 --- /dev/null +++ b/fdio.infra.pxe/docker-nginx/html/ubuntu_focal_amd64/a4:bf:01:55:af:5e/user-data @@ -0,0 +1,83 @@ +#cloud-config +autoinstall: + version: 1 + apt: + geoip: true + preserve_sources_list: false + primary: + - arches: [amd64, i386] + uri: http://ca.archive.ubuntu.com/ubuntu + - arches: [default] + uri: http://ports.ubuntu.com/ubuntu-ports + identity: + hostname: s94-t39-sut2 + password: $6$Y62lhMGJD8YNzmJn$H4DSqjrwFp5WN3tOvIrF.g/G2duOV76zXHAmaA/RU8jfT8H8sDumLQe/Q.EmI5pjPv7dzgI8j9BQPWes7efBK0 + realname: testuser + username: testuser + keyboard: + layout: us + toggle: null + variant: '' + locale: en_US.UTF-8 + network: + ethernets: + eno1: + addresses: [ 10.30.51.94/24 ] + dhcp4: false + gateway4: 10.30.51.1 + nameservers: + addresses: [ 1.1.1.1, 8.8.8.8 ] + version: 2 + packages: + - python3 + refresh-installer: + update: no + ssh: + allow-pw: true + authorized-keys: [] + install-server: true + storage: + config: + - grub_device: false + id: disk-sda + path: /dev/sda + ptable: gpt + preserve: false + type: disk + wipe: superblock-recursive + - device: disk-sda + grub_device: true + id: partition-0 + flag: boot + number: 1 + preserve: false + size: 536870912 + type: partition + wipe: superblock + - id: format-0 + fstype: fat32 + preserve: false + type: format + volume: partition-0 + - device: disk-sda + flag: '' + grub_device: false + id: partition-1 + number: 2 + preserve: false + size: -1 + type: partition + wipe: superblock + - fstype: ext4 + id: format-1 + preserve: false + type: format + volume: partition-1 + - device: format-1 + id: mount-1 + path: / + type: mount + - device: format-0 + id: mount-0 + path: /boot/efi + type: mount \ No newline at end of file -- 2.16.6