resources/tools/disk-image-builder/centos/run-listmaker.sh

   1 #!/bin/bash
   2
   3 # Copyright (c) 2018 Cisco and/or its affiliates.
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at:
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 # This script is to spin up a simulation in VIRL, and fetch the URLs for all packages
  17 # that the user would obtain if they did an "yum update" today.
  18 #
  19 # This entire step is neither secure nor portable. The assumption --for now-- is that
  20 # this will only ever be run in LF CSIT VIRL lab. Should the requirement arise to
  21 # run this elsewhere, then additional work may be required to make this more
  22 # portable.
  23
  24 # This script requires that the following two environment variables be defined-
  25 #
  26 # $VIRL_USER
  27 # $VIRL_PASSWORD
  28
  29 VERSION=$(cat $(dirname $0)/CHANGELOG  | grep '^## ' | head -1 | sed -e 's/.*\[\(.*\)\].*/\1/')
  30 if [ "${VERSION}" = "" ]
  31 then
  32   echo "Unable to determine build version from CHANGELOG file. Make sure"
  33   echo "that there is an entry for the most recent version in CHANGELOG,"
  34   echo "and that the entry is formated like"
  35   echo
  36   echo "## [1.0] - 2016-05-20"
  37   exit 1
  38 fi
  39 DATE=$(date +%Y-%m-%d)
  40
  41 RPMS_TMP_DIR=`mktemp -d`
  42 RPMS_WANTED_FILE=$RPMS_TMP_DIR/rpms_wanted.txt
  43 REPO_MOD_FILE=$RPMS_TMP_DIR/Centos-Vault.repo
  44
  45 if [ "$1" == "centos-7.3-1611" ]
  46 then
  47     OS="centos-7.3-1611"
  48     VIRL_TOPOLOGY_FILE="listmaker/virl-listmaker-centos-7.3-1611.yaml"
  49 echo '
  50 # C7.3.1611
  51 [C7.3.1611-base]
  52 name=CentOS-7.3.1611 - Base
  53 baseurl=http://vault.centos.org/7.3.1611/os/$basearch/
  54 gpgcheck=1
  55 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
  56 enabled=1
  57
  58 [C7.3.1611-updates]
  59 name=CentOS-7.3.1611 - Updates
  60 baseurl=http://vault.centos.org/7.3.1611/updates/$basearch/
  61 gpgcheck=1
  62 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
  63 enabled=1
  64
  65 [C7.3.1611-extras]
  66 name=CentOS-7.3.1611 - Extras
  67 baseurl=http://vault.centos.org/7.3.1611/extras/$basearch/
  68 gpgcheck=1
  69 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
  70 enabled=1
  71
  72 [C7.3.1611-centosplus]
  73 name=CentOS-7.3.1611 - CentOSPlus
  74 baseurl=http://vault.centos.org/7.3.1611/centosplus/$basearch/
  75 gpgcheck=1
  76 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
  77 enabled=0
  78
  79 [C7.3.1611-fasttrack]
  80 name=CentOS-7.3.1611 - CentOSPlus
  81 baseurl=http://vault.centos.org/7.3.1611/fasttrack/$basearch/
  82 gpgcheck=1
  83 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
  84 enabled=0
  85
  86 ' >  $REPO_MOD_FILE
  87 else
  88     if [ "$1" == "centos-7.4-1711" ]
  89         then
  90             OS="centos-7.4-1711"
  91             VIRL_TOPOLOGY_FILE="listmaker/virl-listmaker-centos-7.4-1711.yaml"
  92     elif [ "$1" == "centos-7.6-1810" ]
  93         then
  94             OS="centos-7.6-1810"
  95             VIRL_TOPOLOGY_FILE="listmaker/virl-listmaker-centos-7.6-1810.yaml"
  96     else
  97         echo specify argument -- probably centos-7.3-1611 , centos-7.4-1711 or centos-7.6-1810
  98         exit 1
  99     fi
 100 fi
 101
 102 RELEASE="${OS}_${DATE}_${VERSION}"
 103 OUTPUT_DIR="lists/${RELEASE}"
 104
 105 echo "Building release ${RELEASE}."
 106 echo "Storing data in ${OUTPUT_DIR}/."
 107
 108
 109
 110 # RPM packages wanted
 111
 112 echo '
 113 #RPM_WANTLIST_INFRA
 114 nfs-utils
 115 cloud-init
 116 pkgconfig
 117 yum-utils
 118 #RPM_WANTLIST_CSIT
 119 python-devel
 120 python-virtualenv
 121 python-setuptools
 122 redhat-rpm-config
 123 epel-release
 124 python-srpm-macros
 125 python-rpm-macros
 126 python34
 127 python36-ply
 128 python36-devel
 129 python36-pip
 130 python2-pip-8.1.2-8.el7.noarch http://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/p/
 131 openssl-devel
 132 git
 133 trousers
 134 strongswan-5.7.2-1.el7.x86_64 http://mirror.math.princeton.edu/pub/epel/7/x86_64/Packages/s/
 135 python-cffi
 136 #RPM_WANTLIST_TLDK
 137 tcpdump
 138 #RPM_WANTLIST_VPP
 139 elfutils-libelf
 140 elfutils-libelf-devel
 141 kernel-debug-devel
 142 gcc
 143 dkms-2.6.1-1.el7.noarch https://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/d/
 144 bridge-utils
 145 selinux-policy
 146 selinux-policy-devel
 147 mbedtls-2.7.10-1.el7.x86_64 https://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/m/
 148 mbedtls-devel-2.7.10-1.el7.x86_64 https://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/m/
 149 #RPM_WANTLIST_TREX
 150 zlib-devel
 151 unzip
 152 #RPM_WANTLIST_MISC
 153 socat
 154 psmisc
 155 gperftools
 156 glusterfs
 157 glusterfs-api
 158 libiscsi
 159 libibverbs
 160 libpcap
 161 libpcap-devel
 162 pixman
 163 libpng
 164 pulseaudio-libs
 165 librados2
 166 librbd1
 167 librdmacm
 168 libseccomp
 169 spice-server
 170 spice-server-devel
 171 libusb
 172 usbredir
 173 glusterfs-devel
 174 seavgabios-bin
 175 sgabios-bin
 176 ipxe-roms-qemu
 177 nss-devel
 178 seabios-bin
 179 libffi-devel
 180 boost-filesystem
 181 #RPM_WANTLIST_NESTED
 182 trousers
 183 libnettle
 184 gnutls
 185 libcacard
 186 libcacard-tools
 187 libcacard-devel
 188 device-mapper-multipath-libs
 189 libepoxy
 190 libibumad
 191 qemu-img-ev-2.12.0-18.el7_6.3.1.x86_64 http://mirror.centos.org/centos/7/virt/x86_64/kvm-common/
 192 qemu-kvm-tools-ev-2.12.0-18.el7_6.3.1.x86_64 http://mirror.centos.org/centos/7/virt/x86_64/kvm-common/
 193 qemu-kvm-common-ev-2.12.0-18.el7_6.3.1.x86_64 http://mirror.centos.org/centos/7/virt/x86_64/kvm-common/
 194 qemu-kvm-ev-2.12.0-18.el7_6.3.1.x86_64 http://mirror.centos.org/centos/7/virt/x86_64/kvm-common/
 195 #RPM_WANTLIST_JAVA
 196 java-1.8.0-openjdk-headless
 197 java-1.8.0-openjdk-devel
 198 ' > $RPMS_WANTED_FILE
 199
 200 RPM_OUTPUTFILE="${OUTPUT_DIR}/rpm-packages.txt"
 201 REPO_OUTPUTFILE="${OUTPUT_DIR}/Centos-Vault.repo"
 202
 203 # Python requirements file. Can point to a manually crafted file
 204 # here, or to the actual CSIT requirements file, or to a symlink.
 205
 206 PIP_REQUIREMENTS="../../../../requirements.txt"
 207 if [ ! -f ${PIP_REQUIREMENTS} ]
 208 then
 209   echo "PIP requirements file ${PIP_REQUIREMENTS} not found."
 210   exit 1
 211 fi
 212
 213 PIP_OUTPUTFILE="${OUTPUT_DIR}/pip-requirements.txt"
 214
 215 # These will be used for SSH to the listmaker VM, and must match with what
 216 # was defined in the listmaker VM's kickstart file.
 217 SSH_USER="root"
 218 SSH_PASS="csit"
 219
 220 ###
 221 ### Spin up simulation
 222 ###
 223 if [ "$VIRL_USER" = "" ] || [ "$VIRL_PASSWORD" = "" ]
 224 then
 225   echo '$VIRL_USER and $VIRL_PASSWORD environment variables must be defined'
 226   exit 1
 227 fi
 228
 229 output=$(virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} \
 230   simengine-launch -f ${VIRL_TOPOLOGY_FILE} 2>&1)
 231 id=$(echo "${output}" | grep "Simulation ID is " | cut -f 4 -d ' ')
 232
 233 if [ "$id" = "" ]
 234 then
 235   echo "Did not get a simulation ID. Aborting."
 236   echo "Output was:"
 237   echo "${output}"
 238   exit 1
 239 fi
 240
 241 echo My ID is ${id}
 242 function stop_sim {
 243   virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} simengine-stop --session ${id}
 244 }
 245 trap stop_sim EXIT
 246
 247 ip="None"
 248 while [ "${ip}" = "None" ] || [ "${ip}" = "" ]
 249 do
 250   sleep 5
 251   output=$(virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} simengine-interfaces --session ${id} --nodes listmaker --interfaces management 2>&1)
 252   ip=$(echo "${output}" | grep "u'ip-address" | cut -f 4 -d "'" | cut -f 1 -d '/')
 253 done
 254 echo "IP is $ip"
 255
 256 sleep 10
 257
 258 if ping -w 60 -c 2 $ip > /dev/null
 259 then
 260   echo Host $ip alive
 261 else
 262   echo Host $ip failed to respond to ping
 263   exit 1
 264 fi
 265
 266 # Wait for SSH to be up
 267 while ! nc -z $ip 22
 268 do
 269   sleep 3
 270 done
 271
 272 if [ ! -d ${OUTPUT_DIR} ]; then
 273     mkdir -p $OUTPUT_DIR
 274 fi
 275
 276 ###
 277 ### SSH to the VM and perform package installation. Before each step,
 278 ### dry-run and grab the URLs of the packages that would be installed.
 279 ###
 280
 281 function do_ssh {
 282   # Helper function: SSH and avoid password prompt
 283   sshpass -p $SSH_PASS ssh -o StrictHostKeyChecking=false -o UserKnownHostsFile=/dev/null \
 284     -o LogLevel=error ${SSH_USER}@${ip} "$@"
 285 }
 286
 287 RPM_TEMPFILE=$(mktemp)
 288 ###
 289 ### If there is a repo file specified install it. Freeze yum to release specified above to
 290 ### avoid updating to be packages newer then the specified Centos release. Most packages are
 291 ### installed with yum from a specified Centos version. The packages with urls after them
 292 ### have specific versions and they are installed by rpm from the url.
 293 ###
 294
 295 tmp2=$(mktemp)
 296 echo '#!/bin/bash' > $tmp2
 297
 298 if [ -e ${REPO_MOD_FILE} ] ; then
 299     do_ssh cp /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.back
 300     do_ssh mv /etc/yum.repos.d/CentOS-Vault.repo /etc/yum.repos.d/CentOS-Vault.back
 301
 302     do_ssh "cat - > /tmp/tmp-Vault.repo" < ${REPO_MOD_FILE}
 303     do_ssh cp -f /tmp/tmp-Vault.repo /etc/yum.repos.d/CentOS-Vault.repo
 304
 305     echo "sed -i '/gpgcheck=1/s/.*/&\nenabled=0/' /etc/yum.repos.d/CentOS-Base.repo" >> $tmp2
 306     do_ssh "cat - > /tmp/chrepo.sh" < ${tmp2}
 307     do_ssh chmod +x /tmp/chrepo.sh
 308     do_ssh /tmp/chrepo.sh
 309 fi
 310 PKG_SCRIPT=$(mktemp)
 311 echo \
 312 'while IFS='' read -r line || [[ -n $line ]] ; do
 313     array=( $line )
 314     if [[ -z ${array[0]}  ]] ; then :;
 315     elif [[ ${array[0]:0:1} == "#" ]] ; then :;
 316     else
 317         pkg="${array[0]}"
 318         url="${array[1]}"
 319         if [[ -z $url ]] ; then
 320             yum install -y $pkg
 321             echo $pkg >> /tmp/installedpackages.txt
 322         else
 323             rpm -i --force $url$pkg.rpm
 324             echo "$(rpm -q $pkg) $(echo $url)"  >> /tmp/installedpackages.txt
 325         fi
 326     fi
 327 done < /tmp/rpms-wanted.txt
 328 ' > $PKG_SCRIPT
 329
 330 do_ssh "cat - > /tmp/installpackages.sh" < $PKG_SCRIPT
 331 do_ssh "cat - > /tmp/rpms-wanted.txt" < $RPMS_WANTED_FILE
 332 do_ssh chmod +x /tmp/installpackages.sh
 333 do_ssh /tmp/installpackages.sh
 334
 335 ###
 336 ### Extract package list with versions and urls
 337 ###
 338 sshpass -p "$SSH_PASS" scp -o StrictHostKeyChecking=false -o UserKnownHostsFile=/dev/null  $SSH_USER@${ip}:/tmp/installedpackages.txt $RPM_TEMPFILE
 339
 340 if [ -e ${REPO_MOD_FILE} ] ; then
 341     cp $REPO_MOD_FILE $REPO_OUTPUTFILE
 342 fi
 343 cat $RPM_TEMPFILE | sort > $RPM_OUTPUTFILE
 344 rm -f $RPM_TEMPFILE
 345
 346 ### Get Python data. We do this by installing as per our
 347 ### requirements.txt file while fetching a list of all
 348 ### installed modules before and after, and then comparing.
 349
 350 PIP_TEMPFILE_BEFORE=$(mktemp)
 351 PIP_TEMPFILE_AFTER=$(mktemp)
 352 do_ssh "cat - > /tmp/requirements.txt" < ${PIP_REQUIREMENTS}
 353 do_ssh pip list | sort > $PIP_TEMPFILE_BEFORE
 354 do_ssh pip install -r /tmp/requirements.txt
 355 do_ssh pip list | sort > $PIP_TEMPFILE_AFTER
 356
 357 comm -1 -3 ${PIP_TEMPFILE_BEFORE} ${PIP_TEMPFILE_AFTER} | \
 358   sed -e 's/\(.*\) (\(.*\))/\1==\2/' > $PIP_OUTPUTFILE
 359 rm -f $PIP_TEMPFILE_BEFORE
 360 rm -f $PIP_TEMPFILE_AFTER
 361
 362 ###
 363 ### Stop VIRL session
 364 ###
 365 virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} simengine-stop --session ${id}
 366 trap "" EXIT