From fc11dba19e92609e0631c567ac25756f0c3697df Mon Sep 17 00:00:00 2001 From: Thomas F Herbert Date: Sun, 11 Mar 2018 19:01:09 -0400 Subject: [PATCH] Update to Centos image 7.4-1711 Add qemu-ev version 2.9.0-16 Changes required for vpp 18.04: Add selinux policy rpms at required version. Add mbedtls JIRA: CSIT-566 JIRA: CSIT-680 JIRA: CSIT-681 JIRA: CSIT-602 JIRA: CSIT-608 JIRA: CSIT-603 Update Centos image version 1.8: Point to Vault repo specific to for 7.3-1611 for consistent control of package versions and releases. Fix for pypcap import problem. JIRA: CSIT-991 This fix forces the downloads of the complete package for all the pip python packages to be installed on Centos regardless of the Python metadata on the host. Change-Id: Id651dd113eec41db738ac1e10c92d1074e89b1de Signed-off-by: Thomas F Herbert --- .../tools/disk-image-builder/centos/CHANGELOG | 12 ++ .../disk-image-builder/centos/build-listmaker.sh | 17 +- resources/tools/disk-image-builder/centos/build.sh | 8 +- .../disk-image-builder/centos/centos-7.3-1611.json | 8 +- .../disk-image-builder/centos/centos-7.4-1711.json | 110 ++++++++++ .../centos/listmaker/centos-7.4-1711.json | 48 +++++ .../listmaker/virl-listmaker-centos-7.4-1711.yaml | 12 ++ .../Centos-Vault.repo | 38 ++++ .../pip-requirements.txt | 15 ++ .../rpm-packages.txt | 59 ++++++ .../pip-requirements.txt | 15 ++ .../rpm-packages.txt | 61 ++++++ .../disk-image-builder/centos/run-listmaker.sh | 229 ++++++++++++++++----- .../centos/scripts-remote/post-install.sh | 20 +- 14 files changed, 581 insertions(+), 71 deletions(-) create mode 100644 resources/tools/disk-image-builder/centos/centos-7.4-1711.json create mode 100644 resources/tools/disk-image-builder/centos/listmaker/centos-7.4-1711.json create mode 100644 resources/tools/disk-image-builder/centos/listmaker/virl-listmaker-centos-7.4-1711.yaml create mode 100644 resources/tools/disk-image-builder/centos/lists/centos-7.3-1611_2018-03-17_1.8/Centos-Vault.repo create mode 100644 resources/tools/disk-image-builder/centos/lists/centos-7.3-1611_2018-03-17_1.8/pip-requirements.txt create mode 100644 resources/tools/disk-image-builder/centos/lists/centos-7.3-1611_2018-03-17_1.8/rpm-packages.txt create mode 100644 resources/tools/disk-image-builder/centos/lists/centos-7.4-1711_2018-03-20_1.9/pip-requirements.txt create mode 100644 resources/tools/disk-image-builder/centos/lists/centos-7.4-1711_2018-03-20_1.9/rpm-packages.txt diff --git a/resources/tools/disk-image-builder/centos/CHANGELOG b/resources/tools/disk-image-builder/centos/CHANGELOG index 2013bfda77..d9bedc04c9 100644 --- a/resources/tools/disk-image-builder/centos/CHANGELOG +++ b/resources/tools/disk-image-builder/centos/CHANGELOG @@ -1,3 +1,15 @@ +## [1.9] - 2018-03-20 + +Centos image 7.4-1711 Add mbedtls and selinux-policy. + +## [1.8] - 2018-03-17 + +Fix pypcap package insallation. Add libffi-devel. + +## [1.7] - 2018-03-10 + +Add mbedtls and selinux-policy packages. Use Centos Vault repo for 7.3.1611. + ## [1.6] - 2017-09-22 Moved IPv6_disable from GRUB boot params to sysctl diff --git a/resources/tools/disk-image-builder/centos/build-listmaker.sh b/resources/tools/disk-image-builder/centos/build-listmaker.sh index 52aaff0b3c..6aa5493a58 100755 --- a/resources/tools/disk-image-builder/centos/build-listmaker.sh +++ b/resources/tools/disk-image-builder/centos/build-listmaker.sh @@ -23,13 +23,18 @@ OUT_DIR="${BUILD_DIR}/output/listmaker" if [ "$1" = "centos-7-1511" ]; then RELEASE_NAME="csit-centos-7-1511-listmaker" PACKER_TEMPLATE="listmaker/centos-7-1511.json" -elif [ "$1" = "centos-7.3-1611" ]; then - RELEASE_NAME="csit-centos-7.3-1611-listmaker" - PACKER_TEMPLATE="listmaker/centos-7.3-1611.json" else - echo "Please provide OS as parameter:" - echo "Options: ${0} [centos-7-1511|centos-7.3-1611]" - exit 1 + if [ "$1" = "centos-7.3-1611" ]; then + RELEASE_NAME="csit-centos-7.3-1611-listmaker" + PACKER_TEMPLATE="listmaker/centos-7.3-1611.json" + elif [ "$1" = "centos-7.4-1711" ]; then + RELEASE_NAME="csit-centos-7.4-1711-listmaker" + PACKER_TEMPLATE="listmaker/centos-7.4-1711.json" + else + echo "Please provide OS as parameter:" + echo "Options: ${0} [centos-7-1511|centos-7.3-1611|centos-7.4-1711]" + exit 1 + fi fi VIRL_IMAGE_SUBTYPE=server diff --git a/resources/tools/disk-image-builder/centos/build.sh b/resources/tools/disk-image-builder/centos/build.sh index c01d8f634e..0276df8d4f 100755 --- a/resources/tools/disk-image-builder/centos/build.sh +++ b/resources/tools/disk-image-builder/centos/build.sh @@ -83,7 +83,7 @@ PACKER_DIR="${BUILD_DIR}/packer" RPM_CACHE_DIR="${BUILD_DIR}/cache/rpm" PIP_CACHE_DIR="${BUILD_DIR}/cache/pip" -PACKER_TEMPLATE="centos-7.3-1611.json" +PACKER_TEMPLATE="$1.json" LISTS_DIR="$(dirname $0)/lists" function syntax { @@ -196,11 +196,17 @@ mkdir -p ${OUTPUT_DIR}/temp/rpm mkdir -p ${RPM_CACHE_DIR} RPM_FILE="${LIST}/rpm-packages.txt" +REPO_FILE="${LIST}/Centos-Vault.repo" ### ### Copy rpm package list to cache dir because we are going to use yum on the image ### echo cp $RPM_FILE ${RPM_CACHE_DIR} cp $RPM_FILE ${RPM_CACHE_DIR} +if [ -e $REPO_FILE ] ; then + echo cp $REPO_FILE ${RPM_CACHE_DIR} + cp $REPO_FILE ${RPM_CACHE_DIR} + ln ${RPM_CACHE_DIR}/Centos-Vault.repo ${OUTPUT_DIR}/temp/rpm/Centos-Vault.repo +fi ln ${RPM_CACHE_DIR}/rpm-packages.txt ${OUTPUT_DIR}/temp/rpm/rpm-packages.txt ## PIP diff --git a/resources/tools/disk-image-builder/centos/centos-7.3-1611.json b/resources/tools/disk-image-builder/centos/centos-7.3-1611.json index a747df9938..350a0a4c76 100644 --- a/resources/tools/disk-image-builder/centos/centos-7.3-1611.json +++ b/resources/tools/disk-image-builder/centos/centos-7.3-1611.json @@ -1,6 +1,6 @@ { - "_c": "CentOS-7 ISO URL at the time of creation of this file (08/2017)", - "_c": "was http://buildlogs.centos.org/rolling/7/isos/x86_64/CentOS-7-x86_64-Minimal.iso.", + "_c": "Corrected CentOS-7 ISO URL to the iso with matching sha1sum of original iso", + "_c": "to https://buildlogs.centos.org/monthly/7/CentOS-7-x86_64-Minimal-1706-01.iso.", "_c": "", "variables": { "release": null, @@ -15,7 +15,7 @@ "name": "{{user `release`}}-virtualbox", "type": "virtualbox-iso", "guest_os_type": "RedHat_64", - "iso_url": "http://buildlogs.centos.org/rolling/7/isos/x86_64/CentOS-7-x86_64-Minimal.iso", + "iso_url": "https://buildlogs.centos.org/monthly/7/CentOS-7-x86_64-Minimal-1706-01.iso", "iso_checksum": "c93c4a9755ebff419ac0a4d5052ed1f8b6fb84f9", "iso_checksum_type": "sha1", "output_directory": "{{user `output_dir`}}", @@ -35,7 +35,7 @@ { "name": "{{user `release`}}-qemu", "type": "qemu", - "iso_url": "http://buildlogs.centos.org/rolling/7/isos/x86_64/CentOS-7-x86_64-Minimal.iso", + "iso_url": "https://buildlogs.centos.org/monthly/7/CentOS-7-x86_64-Minimal-1706-01.iso", "iso_checksum": "c93c4a9755ebff419ac0a4d5052ed1f8b6fb84f9", "iso_checksum_type": "sha1", "output_directory": "{{user `output_dir`}}", diff --git a/resources/tools/disk-image-builder/centos/centos-7.4-1711.json b/resources/tools/disk-image-builder/centos/centos-7.4-1711.json new file mode 100644 index 0000000000..056e98ca3c --- /dev/null +++ b/resources/tools/disk-image-builder/centos/centos-7.4-1711.json @@ -0,0 +1,110 @@ +{ + "_c": "CentOS-7 ISO URL at the time of creation of this file (08/2017)", + "_c": "was http://buildlogs.centos.org/rolling/7/isos/x86_64/CentOS-7-x86_64-Minimal.iso.", + "_c": "", + "variables": { + "release": null, + "output_dir": null, + "temp_dir": null, + "atlas_token": "{{env `ATLAS_TOKEN`}}", + "virl_user": "{{env `VIRL_USER`}}", + "virl_password": "{{env `VIRL_PASSWORD`}}" + }, + "builders": [ + { + "name": "{{user `release`}}-virtualbox", + "type": "virtualbox-iso", + "guest_os_type": "RedHat_64", + "iso_url": "https://buildlogs.centos.org/rolling/7/isos/x86_64/CentOS-7-x86_64-Minimal-1711-01.iso", + "iso_checksum": "e892ff40c0c9a32dff403d4ef8b14f246a954f91", + "iso_checksum_type": "sha1", + "output_directory": "{{user `output_dir`}}", + "ssh_username": "root", + "ssh_password": "csit", + "ssh_wait_timeout": "30m", + "disk_size": "16384", + "shutdown_command": "echo 'packer' | sudo -S /sbin/halt -h -p", + "http_directory": "html", + "iso_target_path" : "build/CentOS-7-x86_64-Minimal-1711-01.iso", + "headless": "true", + "boot_command": + [ + " text ks=http://{{ .HTTPIP }}:{{ .HTTPPort }}/ks.cfg" + ] + }, + { + "name": "{{user `release`}}-qemu", + "type": "qemu", + "iso_url": "https://buildlogs.centos.org/rolling/7/isos/x86_64/CentOS-7-x86_64-Minimal-1711-01.iso", + "iso_checksum": "e892ff40c0c9a32dff403d4ef8b14f246a954f91", + "iso_checksum_type": "sha1", + "output_directory": "{{user `output_dir`}}", + "ssh_username": "root", + "ssh_password": "csit", + "ssh_wait_timeout": "30m", + "disk_size": "16384", + "shutdown_command": "echo 'packer' | sudo -S /sbin/halt -h -p", + "http_directory": "html", + "iso_target_path" : "build/CentOS-7-x86_64-Minimal-1711-01.iso", + "headless": "true", + "boot_command": + [ + " text ks=http://{{ .HTTPIP }}:{{ .HTTPPort }}/ks.cfg" + ] + } + ], + "provisioners": [ + { + "type": "file", + "source": "{{ user `temp_dir`}}", + "destination": "/root" + }, + { + "type": "shell", + "script": "scripts-remote/post-install.sh" + }, + { + "type": "shell", + "only": ["{{user `release`}}-qemu"], + "script": "scripts-remote/serial-console-centos-7-1511.sh" + }, + { + "type": "shell", + "only": ["{{user `release`}}-virtualbox"], + "script": "scripts-remote/vagrant-user.sh" + }, + { + "type": "shell", + "only": ["{{user `release`}}-virtualbox"], + "script": "scripts-remote/vagrant-guestadditions.sh" + }, + { + "type": "shell", + "script": "scripts-remote/disable_ipv6.sh" + }, + { + "type": "shell", + "script": "scripts-remote/cleanup.sh" + } + ], + "post-processors": [ + { + "type": "vagrant", + "only": ["{{user `release`}}-virtualbox"], + "keep_input_artifact": true + }, + { + "type": "shell-local", + "only": ["{{user `release`}}-qemu"], + "script": "scripts-local/upload-image-to-virl.sh", + "keep_input_artifact": true, + "environment_vars": + [ + "VIRL_USER={{user `virl_user`}}", + "VIRL_PASSWORD={{user `virl_password`}}", + "VIRL_IMAGE_SUBTYPE=server", + "VIRL_IMAGE_NAME={{user `release`}}" + ] + } + ] +} diff --git a/resources/tools/disk-image-builder/centos/listmaker/centos-7.4-1711.json b/resources/tools/disk-image-builder/centos/listmaker/centos-7.4-1711.json new file mode 100644 index 0000000000..be2d4158ba --- /dev/null +++ b/resources/tools/disk-image-builder/centos/listmaker/centos-7.4-1711.json @@ -0,0 +1,48 @@ +{ + "_c": "Centos7 ISO URL at the time of creation of this file (8/2017)", + "_c": "http://buildlogs.centos.org/rolling/7/isos/x86_64/CentOS-7-x86_64-Minimal.iso", + "_c": "", + "variables": { + "release": null, + "outputdir": null, + "atlas_token": "{{env `ATLAS_TOKEN`}}", + "virl_user": "{{env `VIRL_USER`}}", + "virl_password": "{{env `VIRL_PASSWORD`}}" + }, + "builders": [ + { + "name": "{{user `release`}}", + "type": "qemu", + "iso_url": "https://buildlogs.centos.org/rolling/7/isos/x86_64/CentOS-7-x86_64-Minimal-1711-01.iso", + "iso_checksum": "e892ff40c0c9a32dff403d4ef8b14f246a954f91", + "iso_checksum_type": "sha1", + "output_directory": "{{user `outputdir`}}", + "ssh_username": "root", + "ssh_password": "csit", + "ssh_wait_timeout": "30m", + "disk_size": "16384", + "shutdown_command": "shutdown -P now", + "http_directory": "html", + "iso_target_path" : "build/CentOS-7-x86_64-Minimal-1711-01.iso", + "headless": "true", + "boot_command": + [ + " text ks=http://{{ .HTTPIP }}:{{ .HTTPPort }}/ks.cfg" + ] + } + ], + "post-processors": [ + { + "type": "shell-local", + "script": "scripts-local/upload-image-to-virl.sh", + "keep_input_artifact": true, + "environment_vars": + [ + "VIRL_USER={{user `virl_user`}}", + "VIRL_PASSWORD={{user `virl_password`}}", + "VIRL_IMAGE_SUBTYPE=server", + "VIRL_IMAGE_NAME={{user `release`}}" + ] + } + ] +} diff --git a/resources/tools/disk-image-builder/centos/listmaker/virl-listmaker-centos-7.4-1711.yaml b/resources/tools/disk-image-builder/centos/listmaker/virl-listmaker-centos-7.4-1711.yaml new file mode 100644 index 0000000000..3583f7c8e1 --- /dev/null +++ b/resources/tools/disk-image-builder/centos/listmaker/virl-listmaker-centos-7.4-1711.yaml @@ -0,0 +1,12 @@ + + + + flat + + + + UNUSED + false + + + diff --git a/resources/tools/disk-image-builder/centos/lists/centos-7.3-1611_2018-03-17_1.8/Centos-Vault.repo b/resources/tools/disk-image-builder/centos/lists/centos-7.3-1611_2018-03-17_1.8/Centos-Vault.repo new file mode 100644 index 0000000000..0f1627a6f5 --- /dev/null +++ b/resources/tools/disk-image-builder/centos/lists/centos-7.3-1611_2018-03-17_1.8/Centos-Vault.repo @@ -0,0 +1,38 @@ + +# C7.3.1611 +[C7.3.1611-base] +name=CentOS-7.3.1611 - Base +baseurl=http://vault.centos.org/7.3.1611/os/$basearch/ +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 +enabled=1 + +[C7.3.1611-updates] +name=CentOS-7.3.1611 - Updates +baseurl=http://vault.centos.org/7.3.1611/updates/$basearch/ +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 +enabled=1 + +[C7.3.1611-extras] +name=CentOS-7.3.1611 - Extras +baseurl=http://vault.centos.org/7.3.1611/extras/$basearch/ +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 +enabled=1 + +[C7.3.1611-centosplus] +name=CentOS-7.3.1611 - CentOSPlus +baseurl=http://vault.centos.org/7.3.1611/centosplus/$basearch/ +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 +enabled=0 + +[C7.3.1611-fasttrack] +name=CentOS-7.3.1611 - CentOSPlus +baseurl=http://vault.centos.org/7.3.1611/fasttrack/$basearch/ +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 +enabled=0 + + diff --git a/resources/tools/disk-image-builder/centos/lists/centos-7.3-1611_2018-03-17_1.8/pip-requirements.txt b/resources/tools/disk-image-builder/centos/lists/centos-7.3-1611_2018-03-17_1.8/pip-requirements.txt new file mode 100644 index 0000000000..e0d4c1cfa6 --- /dev/null +++ b/resources/tools/disk-image-builder/centos/lists/centos-7.3-1611_2018-03-17_1.8/pip-requirements.txt @@ -0,0 +1,15 @@ +docopt==0.6.2 +ecdsa==0.13 +enum34==1.1.2 +interruptingcow==0.6 +ipaddress==1.0.16 +paramiko==1.16.0 +pycrypto==2.6.1 +pykwalify==1.5.0 +pypcap==1.1.5 +python-dateutil==2.4.2 +PyYAML==3.11 +requests==2.9.1 +robotframework==2.9.2 +scapy==2.3.1 +scp==0.10.2 diff --git a/resources/tools/disk-image-builder/centos/lists/centos-7.3-1611_2018-03-17_1.8/rpm-packages.txt b/resources/tools/disk-image-builder/centos/lists/centos-7.3-1611_2018-03-17_1.8/rpm-packages.txt new file mode 100644 index 0000000000..76872b10b3 --- /dev/null +++ b/resources/tools/disk-image-builder/centos/lists/centos-7.3-1611_2018-03-17_1.8/rpm-packages.txt @@ -0,0 +1,59 @@ +bridge-utils +cloud-init +dkms-2.4.0-1.20170926git959bd74.el7.noarch http://mirror.math.princeton.edu/pub/epel/7/x86_64/Packages/d/ +elfutils-libelf +elfutils-libelf-devel +gcc +git +glusterfs +glusterfs-api +glusterfs-devel +gperftools +ipxe-roms-qemu +java-1.8.0-openjdk-devel +java-1.8.0-openjdk-headless +kernel-debug-devel +libcacard-devel-ev-2.3.0-31.el7_2.21.1.x86_64 http://cbs.centos.org/kojifiles/packages/qemu-kvm-ev/2.3.0/31.el7_2.21.1/x86_64/ +libcacard-ev-2.3.0-31.el7_2.21.1.x86_64 http://cbs.centos.org/kojifiles/packages/qemu-kvm-ev/2.3.0/31.el7_2.21.1/x86_64/ +libcacard-tools-ev-2.3.0-31.el7_2.21.1.x86_64 http://cbs.centos.org/kojifiles/packages/qemu-kvm-ev/2.3.0/31.el7_2.21.1/x86_64/ +libffi-devel +libibverbs +libiscsi +libpcap +libpcap-devel +libpng +librados2 +librbd1 +librdmacm +libseccomp +libusb +mbedtls-2.7.0-1.el7.x86_64 http://mirror.math.princeton.edu/pub/epel/7/x86_64/Packages/m/ +nfs-utils +nss-devel +openssl-devel +pixman +pkgconfig +psmisc +pulseaudio-libs +python2-pip-8.1.2-5.el7.noarch http://mirror.math.princeton.edu/pub/epel/7/x86_64/Packages/p/ +python-devel +python-setuptools +python-virtualenv +qemu-img-ev-2.3.0-31.el7_2.21.1.x86_64 http://cbs.centos.org/kojifiles/packages/qemu-kvm-ev/2.3.0/31.el7_2.21.1/x86_64/ +qemu-kvm-common-ev-2.3.0-31.el7_2.21.1.x86_64 http://cbs.centos.org/kojifiles/packages/qemu-kvm-ev/2.3.0/31.el7_2.21.1/x86_64/ +qemu-kvm-ev-2.3.0-31.el7_2.21.1.x86_64 http://cbs.centos.org/kojifiles/packages/qemu-kvm-ev/2.3.0/31.el7_2.21.1/x86_64/ +qemu-kvm-ev-debuginfo-2.3.0-31.el7_2.21.1.x86_64 http://cbs.centos.org/kojifiles/packages/qemu-kvm-ev/2.3.0/31.el7_2.21.1/x86_64/ +qemu-kvm-tools-ev-2.3.0-31.el7_2.21.1.x86_64 http://cbs.centos.org/kojifiles/packages/qemu-kvm-ev/2.3.0/31.el7_2.21.1/x86_64/ +seabios-bin +seavgabios-bin +selinux-policy +selinux-policy-devel +sgabios-bin +socat +spice-server +strongswan-5.6.1-2.el7.x86_64 http://mirror.math.princeton.edu/pub/epel/7/x86_64/Packages/s/ +tcpdump +unzip +usbredir +yum-utils +zlib-devel diff --git a/resources/tools/disk-image-builder/centos/lists/centos-7.4-1711_2018-03-20_1.9/pip-requirements.txt b/resources/tools/disk-image-builder/centos/lists/centos-7.4-1711_2018-03-20_1.9/pip-requirements.txt new file mode 100644 index 0000000000..e0d4c1cfa6 --- /dev/null +++ b/resources/tools/disk-image-builder/centos/lists/centos-7.4-1711_2018-03-20_1.9/pip-requirements.txt @@ -0,0 +1,15 @@ +docopt==0.6.2 +ecdsa==0.13 +enum34==1.1.2 +interruptingcow==0.6 +ipaddress==1.0.16 +paramiko==1.16.0 +pycrypto==2.6.1 +pykwalify==1.5.0 +pypcap==1.1.5 +python-dateutil==2.4.2 +PyYAML==3.11 +requests==2.9.1 +robotframework==2.9.2 +scapy==2.3.1 +scp==0.10.2 diff --git a/resources/tools/disk-image-builder/centos/lists/centos-7.4-1711_2018-03-20_1.9/rpm-packages.txt b/resources/tools/disk-image-builder/centos/lists/centos-7.4-1711_2018-03-20_1.9/rpm-packages.txt new file mode 100644 index 0000000000..044fa601e7 --- /dev/null +++ b/resources/tools/disk-image-builder/centos/lists/centos-7.4-1711_2018-03-20_1.9/rpm-packages.txt @@ -0,0 +1,61 @@ +bridge-utils +cloud-init +dkms-2.4.0-1.20170926git959bd74.el7.noarch http://mirror.math.princeton.edu/pub/epel/7/x86_64/Packages/d/ +elfutils-libelf +elfutils-libelf-devel +gcc +git +glusterfs +glusterfs-api +glusterfs-devel +gnutls +gperftools +ipxe-roms-qemu +java-1.8.0-openjdk-devel +java-1.8.0-openjdk-headless +kernel-debug-devel +libcacard +libcacard-devel +libcacard-tools +libffi-devel +libibverbs +libiscsi +libnettle +libpcap +libpcap-devel +libpng +librados2 +librbd1 +librdmacm +libseccomp +libusb +mbedtls-2.7.0-1.el7.x86_64 http://mirror.math.princeton.edu/pub/epel/7/x86_64/Packages/m/ +nfs-utils +nss-devel +openssl-devel +package strongswan-5.6.1-2.el7.x86_64 is not installed http://mirror.math.princeton.edu/pub/epel/7/x86_64/Packages/s/ +pixman +pkgconfig +psmisc +pulseaudio-libs +python2-pip-8.1.2-5.el7.noarch http://mirror.math.princeton.edu/pub/epel/7/x86_64/Packages/p/ +python-devel +python-setuptools +python-virtualenv +qemu-img-ev-2.9.0-16.el7_4.13.1.x86_64 http://mirror.centos.org/centos-7/7/virt/x86_64/kvm-common/ +qemu-kvm-common-ev-2.9.0-16.el7_4.13.1.x86_64 http://mirror.centos.org/centos-7/7/virt/x86_64/kvm-common/ +qemu-kvm-ev-2.9.0-16.el7_4.13.1.x86_64 http://mirror.centos.org/centos-7/7/virt/x86_64/kvm-common/ +qemu-kvm-tools-ev-2.9.0-16.el7_4.13.1.x86_64 http://mirror.centos.org/centos-7/7/virt/x86_64/kvm-common/ +seabios-bin +seavgabios-bin +selinux-policy +selinux-policy-devel +sgabios-bin +socat +spice-server +tcpdump +trousers +unzip +usbredir +yum-utils +zlib-devel diff --git a/resources/tools/disk-image-builder/centos/run-listmaker.sh b/resources/tools/disk-image-builder/centos/run-listmaker.sh index 24463c48da..b94f9fa3de 100755 --- a/resources/tools/disk-image-builder/centos/run-listmaker.sh +++ b/resources/tools/disk-image-builder/centos/run-listmaker.sh @@ -38,17 +38,65 @@ then fi DATE=$(date +%Y-%m-%d) +RPMS_TMP_DIR=`mktemp -d` +RPMS_WANTED_FILE=$RPMS_TMP_DIR/rpms_wanted.txt +REPO_MOD_FILE=$RPMS_TMP_DIR/Centos-Vault.repo + if [ "$1" == "centos-7-1511" ] then OS="centos-7-1511" VIRL_TOPOLOGY_FILE="listmaker/virl-listmaker-centos-7-1511.yaml" -elif [ "$1" == "centos-7.3-1611" ] -then - OS="centos-7.3-1611" - VIRL_TOPOLOGY_FILE="listmaker/virl-listmaker-centos-7.3-1611.yaml" else - echo specify argument -- probably centos-7-1511 or centos-7.3-1611 - exit 1 + if [ "$1" == "centos-7.3-1611" ] + then + OS="centos-7.3-1611" + VIRL_TOPOLOGY_FILE="listmaker/virl-listmaker-centos-7.3-1611.yaml" + echo ' +# C7.3.1611 +[C7.3.1611-base] +name=CentOS-7.3.1611 - Base +baseurl=http://vault.centos.org/7.3.1611/os/$basearch/ +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 +enabled=1 + +[C7.3.1611-updates] +name=CentOS-7.3.1611 - Updates +baseurl=http://vault.centos.org/7.3.1611/updates/$basearch/ +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 +enabled=1 + +[C7.3.1611-extras] +name=CentOS-7.3.1611 - Extras +baseurl=http://vault.centos.org/7.3.1611/extras/$basearch/ +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 +enabled=1 + +[C7.3.1611-centosplus] +name=CentOS-7.3.1611 - CentOSPlus +baseurl=http://vault.centos.org/7.3.1611/centosplus/$basearch/ +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 +enabled=0 + +[C7.3.1611-fasttrack] +name=CentOS-7.3.1611 - CentOSPlus +baseurl=http://vault.centos.org/7.3.1611/fasttrack/$basearch/ +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 +enabled=0 + +' > $REPO_MOD_FILE + elif [ "$1" == "centos-7.4-1711" ] + then + OS="centos-7.4-1711" + VIRL_TOPOLOGY_FILE="listmaker/virl-listmaker-centos-7.4-1711.yaml" + else + echo specify argument -- probably centos-7-1511 or centos-7.3-1611 + exit 1 + fi fi RELEASE="${OS}_${DATE}_${VERSION}" @@ -58,25 +106,83 @@ echo "Building release ${RELEASE}." echo "Storing data in ${OUTPUT_DIR}/." -# RPM packages wanted -RPM_WANTLIST_INFRA="nfs-utils cloud-init pkgconfig yum-utils" -RPM_WANTLIST_CSIT="python-devel python-pip python-virtualenv python-setuptools python-pip openssl-devel git strongswan" -RPM_WANTLIST_TLDK="tcpdump" -RPM_WANTLIST_VPP="dkms bridge-utils libmbedcrypto0 libmbedtls10 libmbedx509-0" -RPM_WANTLIST_TREX="zlib-devel unzip" -RPM_WANTLIST_MISC="socat psmisc gperftools glusterfs glusterfs-api libiscsi libibverbs libpcap libpcap-devel pixman libpng pulseaudio-libs librados2 librbd1 librdmacm libseccomp spice-server libusb usbredir glusterfs-devel seavgabios-bin sgabios-bin ipxe-roms-qemu nss-devel seabios-bin" - -RPM_WANTLIST_NESTED="qemu-img-ev-2.3.0-31.el7_2.21.1.x86_64.rpm libcacard-ev-2.3.0-31.el7_2.21.1.x86_64.rpm libcacard-devel-ev-2.3.0-31.el7_2.21.1.x86_64.rpm qemu-kvm-ev-debuginfo-2.3.0-31.el7_2.21.1.x86_64.rpm qemu-kvm-tools-ev-2.3.0-31.el7_2.21.1.x86_64.rpm qemu-kvm-common-ev-2.3.0-31.el7_2.21.1.x86_64.rpm qemu-kvm-ev-2.3.0-31.el7_2.21.1.x86_64.rpm libcacard-tools-ev-2.3.0-31.el7_2.21.1.x86_64.rpm" -RPM_WANTLIST_JAVA="java-1.8.0-openjdk-headless java-1.8.0-openjdk-devel" -#RPM_WANTLIST_DOCKER="docker-engine" +# RPM packages wanted -### For now, do not include WANTLIST_NESTED in the main list. We're installing qemu -### separately because of the possible need for specific versions but the supported version seem to be ok for Centos 7.3 -## -RPM_WANTLIST="$RPM_WANTLIST_INFRA $RPM_WANTLIST_CSIT $RPM_WANTLIST_VPP $RPM_WANTLIST_TREX $RPM_WANTLIST_MISC $RPM_WANTLIST_JAVA $RPM_WANTLIST_TLDK" +echo ' +#RPM_WANTLIST_INFRA +nfs-utils +cloud-init +pkgconfig +yum-utils +#RPM_WANTLIST_CSIT +python-devel +python-virtualenv +python-setuptools +python2-pip-8.1.2-5.el7.noarch http://mirror.math.princeton.edu/pub/epel/7/x86_64/Packages/p/ +openssl-devel +git +strongswan-5.6.1-2.el7.x86_64 http://mirror.math.princeton.edu/pub/epel/7/x86_64/Packages/s/ +#RPM_WANTLIST_TLDK +tcpdump +#RPM_WANTLIST_VPP +elfutils-libelf +elfutils-libelf-devel +kernel-debug-devel +gcc +dkms-2.4.0-1.20170926git959bd74.el7.noarch http://mirror.math.princeton.edu/pub/epel/7/x86_64/Packages/d/ +bridge-utils +selinux-policy +selinux-policy-devel +mbedtls-2.7.0-1.el7.x86_64 http://mirror.math.princeton.edu/pub/epel/7/x86_64/Packages/m/ +#RPM_WANTLIST_TREX +zlib-devel +unzip +#RPM_WANTLIST_MISC +socat +psmisc +gperftools +glusterfs +glusterfs-api +libiscsi +libibverbs +libpcap +libpcap-devel +pixman +libpng +pulseaudio-libs +librados2 +librbd1 +librdmacm +libseccomp +spice-server +libusb +usbredir +glusterfs-devel +seavgabios-bin +sgabios-bin +ipxe-roms-qemu +nss-devel +seabios-bin +libffi-devel +#RPM_WANTLIST_NESTED +trousers +libnettle +gnutls +libcacard +libcacard-tools +libcacard-devel +qemu-img-ev-2.9.0-16.el7_4.13.1.x86_64 http://mirror.centos.org/centos-7/7/virt/x86_64/kvm-common/ +qemu-kvm-tools-ev-2.9.0-16.el7_4.13.1.x86_64 http://mirror.centos.org/centos-7/7/virt/x86_64/kvm-common/ +qemu-kvm-common-ev-2.9.0-16.el7_4.13.1.x86_64 http://mirror.centos.org/centos-7/7/virt/x86_64/kvm-common/ +qemu-kvm-ev-2.9.0-16.el7_4.13.1.x86_64 http://mirror.centos.org/centos-7/7/virt/x86_64/kvm-common/ +#RPM_WANTLIST_JAVA +java-1.8.0-openjdk-headless +java-1.8.0-openjdk-devel +' > $RPMS_WANTED_FILE RPM_OUTPUTFILE="${OUTPUT_DIR}/rpm-packages.txt" +REPO_OUTPUTFILE="${OUTPUT_DIR}/Centos-Vault.repo" # Python requirements file. Can point to a manually crafted file # here, or to the actual CSIT requirements file, or to a symlink. @@ -163,42 +269,63 @@ function do_ssh { } RPM_TEMPFILE=$(mktemp) -RPM_TEMPFILE2=$(mktemp) -RPM_URL_TEMPFILE=$(mktemp) -do_ssh yum clean all -do_ssh yum install -y @base -do_ssh yum install -y deltarpm -do_ssh yum update -y -do_ssh yum -y install epel-release -do_ssh yum update -y -do_ssh yum -y install $RPM_WANTLIST -for i in ${RPM_WANTLIST} ; do - echo $i >> $RPM_TEMPFILE -done - ### -### Install qemu ($RPM_WANTLIST_NESTED) separately from PPA in case specific versions are required. +### If there is a repo file specified install it. Freeze yum to release specified above to +### avoid updating to be packages newer then the specified Centos release. Most packages are +### installed with yum from a specified Centos version. The packages with urls after them +### have specific versions and they are installed by rpm from the url. ### -for i in ${RPM_WANTLIST_NESTED}; do - echo $i http://cbs.centos.org/kojifiles/packages/qemu-kvm-ev/2.3.0/31.el7_2.21.1/x86_64/ >> $RPM_URL_TEMPFILE -done + +tmp2=$(mktemp) +echo '#!/bin/bash' > $tmp2 + +if [ -e ${REPO_MOD_FILE} ] ; then + do_ssh cp /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.back + do_ssh mv /etc/yum.repos.d/CentOS-Vault.repo /etc/yum.repos.d/CentOS-Vault.back + + do_ssh "cat - > /tmp/tmp-Vault.repo" < ${REPO_MOD_FILE} + do_ssh cp -f /tmp/tmp-Vault.repo /etc/yum.repos.d/CentOS-Vault.repo + + echo "sed -i '/gpgcheck=1/s/.*/&\nenabled=0/' /etc/yum.repos.d/CentOS-Base.repo" >> $tmp2 + do_ssh "cat - > /tmp/chrepo.sh" < ${tmp2} + do_ssh chmod +x /tmp/chrepo.sh + do_ssh /tmp/chrepo.sh +fi +PKG_SCRIPT=$(mktemp) +echo \ +'while IFS='' read -r line || [[ -n $line ]] ; do + array=( $line ) + if [[ -z ${array[0]} ]] ; then :; + elif [[ ${array[0]:0:1} == "#" ]] ; then :; + else + pkg="${array[0]}" + url="${array[1]}" + if [[ -z $url ]] ; then + yum install -y $pkg + echo $pkg >> /tmp/installedpackages.txt + else + rpm -i --force $url$pkg.rpm + echo "$(rpm -q $pkg) $(echo $url)" >> /tmp/installedpackages.txt + fi + fi +done < /tmp/rpms-wanted.txt +' > $PKG_SCRIPT + +do_ssh "cat - > /tmp/installpackages.sh" < $PKG_SCRIPT +do_ssh "cat - > /tmp/rpms-wanted.txt" < $RPMS_WANTED_FILE +do_ssh chmod +x /tmp/installpackages.sh +do_ssh /tmp/installpackages.sh + ### -### Try 2 times for dependencies. Not in yum repo so it is not automatic" +### Extract package list with versions and urls ### -for i in ${RPM_WANTLIST_NESTED}; do - do_ssh rpm -i http://cbs.centos.org/kojifiles/packages/qemu-kvm-ev/2.3.0/31.el7_2.21.1/x86_64/$i -done -for i in ${RPM_WANTLIST_NESTED}; do - do_ssh rpm -i http://cbs.centos.org/kojifiles/packages/qemu-kvm-ev/2.3.0/31.el7_2.21.1/x86_64/$i -done +sshpass -p "$SSH_PASS" scp -o StrictHostKeyChecking=false -o UserKnownHostsFile=/dev/null $SSH_USER@${ip}:/tmp/installedpackages.txt $RPM_TEMPFILE - -cat $RPM_TEMPFILE | sort > $RPM_TEMPFILE2 +if [ -e ${REPO_MOD_FILE} ] ; then + cp $REPO_MOD_FILE $REPO_OUTPUTFILE +fi +cat $RPM_TEMPFILE | sort > $RPM_OUTPUTFILE rm -f $RPM_TEMPFILE -cat $RPM_TEMPFILE2 > $RPM_OUTPUTFILE -cat $RPM_URL_TEMPFILE >> $RPM_OUTPUTFILE -rm -f $RPM_TEMPFILE2 -rm -f $RPM_URL_TEMPFILE ### Get Python data. We do this by installing as per our ### requirements.txt file while fetching a list of all diff --git a/resources/tools/disk-image-builder/centos/scripts-remote/post-install.sh b/resources/tools/disk-image-builder/centos/scripts-remote/post-install.sh index bd45d2330c..fdaeebd58c 100644 --- a/resources/tools/disk-image-builder/centos/scripts-remote/post-install.sh +++ b/resources/tools/disk-image-builder/centos/scripts-remote/post-install.sh @@ -32,17 +32,19 @@ echo =======================End Remove NetworkManager=========================== ### echo "********** INSTALLING RPMs **********" -# We're doing this the hard way as we're dealing with a bunch of -# rpm packages without using yum. +# If it exists, copy in a specific Centos Repo for the Centos version for this image . # Attempt up to five cycles of unpack/configure. There may be dependency # problems during the first one(s). -echo ==========================yum update============================== -yum clean all -yum install -y deltarpm -yum update -y -yum install -y @base epel-release -echo ==========================end yum update============================== +echo ==========================yum repos============================== +if [ -f ${TEMP_PATH}/rpm/Centos-Vault.repo ] ; then + echo ==========================Install specific yum repo====================== + sed -i '/gpgcheck=1/s/.*/&\nenabled=0/' /etc/yum.repos.d/CentOS-Base.repo + cp -f ${TEMP_PATH}/rpm/Centos-Vault.repo /etc/yum.repos.d +else + echo ==========================Update from base repo and updates================ + : ; +fi attempt=1 MAX_ATTEMPTS=3 try_again=1 @@ -55,7 +57,7 @@ do do # use rpm command if url is present in the package file if [ ! -z $url ] ; then - rpm -i $url$name || try_again=1 + rpm -i $url$name.rpm || try_again=1 else yum install -y $name || try_again=1 fi -- 2.16.6