resources/tools/disk-image-builder/centos/run-listmaker.sh

   1 #!/bin/bash
   2
   3 # Copyright (c) 2016 Cisco and/or its affiliates.
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at:
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 # This script is to spin up a simulation in VIRL, and fetch the URLs for all packages
  17 # that the user would obtain if they did an "yum update" today.
  18 #
  19 # This entire step is neither secure nor portable. The assumption --for now-- is that
  20 # this will only ever be run in LF CSIT VIRL lab. Should the requirement arise to
  21 # run this elsewhere, then additional work may be required to make this more
  22 # portable.
  23
  24 # This script requires that the following two environment variables be defined-
  25 #
  26 # $VIRL_USER
  27 # $VIRL_PASSWORD
  28
  29 VERSION=$(cat $(dirname $0)/CHANGELOG  | grep '^## ' | head -1 | sed -e 's/.*\[\(.*\)\].*/\1/')
  30 if [ "${VERSION}" = "" ]
  31 then
  32   echo "Unable to determine build version from CHANGELOG file. Make sure"
  33   echo "that there is an entry for the most recent version in CHANGELOG,"
  34   echo "and that the entry is formated like"
  35   echo
  36   echo "## [1.0] - 2016-05-20"
  37   exit 1
  38 fi
  39 DATE=$(date +%Y-%m-%d)
  40
  41 RPMS_TMP_DIR=`mktemp -d`
  42 RPMS_WANTED_FILE=$RPMS_TMP_DIR/rpms_wanted.txt
  43 REPO_MOD_FILE=$RPMS_TMP_DIR/Centos-Vault.repo
  44
  45 if [ "$1" == "centos-7-1511" ]
  46 then
  47     OS="centos-7-1511"
  48     VIRL_TOPOLOGY_FILE="listmaker/virl-listmaker-centos-7-1511.yaml"
  49 else
  50     if [ "$1" == "centos-7.3-1611" ]
  51     then
  52         OS="centos-7.3-1611"
  53         VIRL_TOPOLOGY_FILE="listmaker/virl-listmaker-centos-7.3-1611.yaml"
  54     echo '
  55 # C7.3.1611
  56 [C7.3.1611-base]
  57 name=CentOS-7.3.1611 - Base
  58 baseurl=http://vault.centos.org/7.3.1611/os/$basearch/
  59 gpgcheck=1
  60 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
  61 enabled=1
  62
  63 [C7.3.1611-updates]
  64 name=CentOS-7.3.1611 - Updates
  65 baseurl=http://vault.centos.org/7.3.1611/updates/$basearch/
  66 gpgcheck=1
  67 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
  68 enabled=1
  69
  70 [C7.3.1611-extras]
  71 name=CentOS-7.3.1611 - Extras
  72 baseurl=http://vault.centos.org/7.3.1611/extras/$basearch/
  73 gpgcheck=1
  74 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
  75 enabled=1
  76
  77 [C7.3.1611-centosplus]
  78 name=CentOS-7.3.1611 - CentOSPlus
  79 baseurl=http://vault.centos.org/7.3.1611/centosplus/$basearch/
  80 gpgcheck=1
  81 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
  82 enabled=0
  83
  84 [C7.3.1611-fasttrack]
  85 name=CentOS-7.3.1611 - CentOSPlus
  86 baseurl=http://vault.centos.org/7.3.1611/fasttrack/$basearch/
  87 gpgcheck=1
  88 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
  89 enabled=0
  90
  91 ' >  $REPO_MOD_FILE
  92     elif [ "$1" == "centos-7.4-1711" ]
  93     then
  94         OS="centos-7.4-1711"
  95         VIRL_TOPOLOGY_FILE="listmaker/virl-listmaker-centos-7.4-1711.yaml"
  96     else
  97         echo specify argument -- probably centos-7-1511 or centos-7.3-1611
  98         exit 1
  99     fi
 100 fi
 101
 102 RELEASE="${OS}_${DATE}_${VERSION}"
 103 OUTPUT_DIR="lists/${RELEASE}"
 104
 105 echo "Building release ${RELEASE}."
 106 echo "Storing data in ${OUTPUT_DIR}/."
 107
 108
 109
 110 # RPM packages wanted
 111
 112 echo '
 113 #RPM_WANTLIST_INFRA
 114 nfs-utils
 115 cloud-init
 116 pkgconfig
 117 yum-utils
 118 #RPM_WANTLIST_CSIT
 119 python-devel
 120 python-virtualenv
 121 python-setuptools
 122 python2-pip-8.1.2-5.el7.noarch http://mirror.math.princeton.edu/pub/epel/7/x86_64/Packages/p/
 123 openssl-devel
 124 git
 125 strongswan-5.6.1-2.el7.x86_64 http://mirror.math.princeton.edu/pub/epel/7/x86_64/Packages/s/
 126 #RPM_WANTLIST_TLDK
 127 tcpdump
 128 #RPM_WANTLIST_VPP
 129 elfutils-libelf
 130 elfutils-libelf-devel
 131 kernel-debug-devel
 132 gcc
 133 dkms-2.4.0-1.20170926git959bd74.el7.noarch http://mirror.math.princeton.edu/pub/epel/7/x86_64/Packages/d/
 134 bridge-utils
 135 selinux-policy
 136 selinux-policy-devel
 137 mbedtls-2.7.0-1.el7.x86_64 http://mirror.math.princeton.edu/pub/epel/7/x86_64/Packages/m/
 138 #RPM_WANTLIST_TREX
 139 zlib-devel
 140 unzip
 141 #RPM_WANTLIST_MISC
 142 socat
 143 psmisc
 144 gperftools
 145 glusterfs
 146 glusterfs-api
 147 libiscsi
 148 libibverbs
 149 libpcap
 150 libpcap-devel
 151 pixman
 152 libpng
 153 pulseaudio-libs
 154 librados2
 155 librbd1
 156 librdmacm
 157 libseccomp
 158 spice-server
 159 libusb
 160 usbredir
 161 glusterfs-devel
 162 seavgabios-bin
 163 sgabios-bin
 164 ipxe-roms-qemu
 165 nss-devel
 166 seabios-bin
 167 libffi-devel
 168 #RPM_WANTLIST_NESTED
 169 trousers
 170 libnettle
 171 gnutls
 172 libcacard
 173 libcacard-tools
 174 libcacard-devel
 175 qemu-img-ev-2.9.0-16.el7_4.13.1.x86_64 http://mirror.centos.org/centos-7/7/virt/x86_64/kvm-common/
 176 qemu-kvm-tools-ev-2.9.0-16.el7_4.13.1.x86_64 http://mirror.centos.org/centos-7/7/virt/x86_64/kvm-common/
 177 qemu-kvm-common-ev-2.9.0-16.el7_4.13.1.x86_64 http://mirror.centos.org/centos-7/7/virt/x86_64/kvm-common/
 178 qemu-kvm-ev-2.9.0-16.el7_4.13.1.x86_64 http://mirror.centos.org/centos-7/7/virt/x86_64/kvm-common/
 179 #RPM_WANTLIST_JAVA
 180 java-1.8.0-openjdk-headless
 181 java-1.8.0-openjdk-devel
 182 ' > $RPMS_WANTED_FILE
 183
 184 RPM_OUTPUTFILE="${OUTPUT_DIR}/rpm-packages.txt"
 185 REPO_OUTPUTFILE="${OUTPUT_DIR}/Centos-Vault.repo"
 186
 187 # Python requirements file. Can point to a manually crafted file
 188 # here, or to the actual CSIT requirements file, or to a symlink.
 189
 190 PIP_REQUIREMENTS="../../../../requirements.txt"
 191 if [ ! -f ${PIP_REQUIREMENTS} ]
 192 then
 193   echo "PIP requirements file ${PIP_REQUIREMENTS} not found."
 194   exit 1
 195 fi
 196
 197 PIP_OUTPUTFILE="${OUTPUT_DIR}/pip-requirements.txt"
 198
 199 # These will be used for SSH to the listmaker VM, and must match with what
 200 # was defined in the listmaker VM's kickstart file.
 201 SSH_USER="root"
 202 SSH_PASS="csit"
 203
 204 ###
 205 ### Spin up simulation
 206 ###
 207 if [ "$VIRL_USER" = "" ] || [ "$VIRL_PASSWORD" = "" ]
 208 then
 209   echo '$VIRL_USER and $VIRL_PASSWORD environment variables must be defined'
 210   exit 1
 211 fi
 212
 213 output=$(virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} \
 214   simengine-launch -f ${VIRL_TOPOLOGY_FILE} 2>&1)
 215 id=$(echo "${output}" | grep "Simulation ID is " | cut -f 4 -d ' ')
 216
 217 if [ "$id" = "" ]
 218 then
 219   echo "Did not get a simulation ID. Aborting."
 220   echo "Output was:"
 221   echo "${output}"
 222   exit 1
 223 fi
 224
 225 echo My ID is ${id}
 226 function stop_sim {
 227   virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} simengine-stop --session ${id}
 228 }
 229 trap stop_sim EXIT
 230
 231 ip="None"
 232 while [ "${ip}" = "None" ] || [ "${ip}" = "" ]
 233 do
 234   sleep 5
 235   output=$(virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} simengine-interfaces --session ${id} --nodes listmaker --interfaces management 2>&1)
 236   ip=$(echo "${output}" | grep "u'ip-address" | cut -f 4 -d "'" | cut -f 1 -d '/')
 237 done
 238 echo "IP is $ip"
 239
 240 sleep 10
 241
 242 if ping -w 60 -c 2 $ip > /dev/null
 243 then
 244   echo Host $ip alive
 245 else
 246   echo Host $ip failed to respond to ping
 247   exit 1
 248 fi
 249
 250 # Wait for SSH to be up
 251 while ! nc -z $ip 22
 252 do
 253   sleep 3
 254 done
 255
 256 if [ ! -d ${OUTPUT_DIR} ]; then
 257     mkdir -p $OUTPUT_DIR
 258 fi
 259
 260 ###
 261 ### SSH to the VM and perform package installation. Before each step,
 262 ### dry-run and grab the URLs of the packages that would be installed.
 263 ###
 264
 265 function do_ssh {
 266   # Helper function: SSH and avoid password prompt
 267   sshpass -p $SSH_PASS ssh -o StrictHostKeyChecking=false -o UserKnownHostsFile=/dev/null \
 268     -o LogLevel=error ${SSH_USER}@${ip} "$@"
 269 }
 270
 271 RPM_TEMPFILE=$(mktemp)
 272 ###
 273 ### If there is a repo file specified install it. Freeze yum to release specified above to
 274 ### avoid updating to be packages newer then the specified Centos release. Most packages are
 275 ### installed with yum from a specified Centos version. The packages with urls after them
 276 ### have specific versions and they are installed by rpm from the url.
 277 ###
 278
 279 tmp2=$(mktemp)
 280 echo '#!/bin/bash' > $tmp2
 281
 282 if [ -e ${REPO_MOD_FILE} ] ; then
 283     do_ssh cp /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.back
 284     do_ssh mv /etc/yum.repos.d/CentOS-Vault.repo /etc/yum.repos.d/CentOS-Vault.back
 285
 286     do_ssh "cat - > /tmp/tmp-Vault.repo" < ${REPO_MOD_FILE}
 287     do_ssh cp -f /tmp/tmp-Vault.repo /etc/yum.repos.d/CentOS-Vault.repo
 288
 289     echo "sed -i '/gpgcheck=1/s/.*/&\nenabled=0/' /etc/yum.repos.d/CentOS-Base.repo" >> $tmp2
 290     do_ssh "cat - > /tmp/chrepo.sh" < ${tmp2}
 291     do_ssh chmod +x /tmp/chrepo.sh
 292     do_ssh /tmp/chrepo.sh
 293 fi
 294 PKG_SCRIPT=$(mktemp)
 295 echo \
 296 'while IFS='' read -r line || [[ -n $line ]] ; do
 297     array=( $line )
 298     if [[ -z ${array[0]}  ]] ; then :;
 299     elif [[ ${array[0]:0:1} == "#" ]] ; then :;
 300     else
 301         pkg="${array[0]}"
 302         url="${array[1]}"
 303         if [[ -z $url ]] ; then
 304             yum install -y $pkg
 305             echo $pkg >> /tmp/installedpackages.txt
 306         else
 307             rpm -i --force $url$pkg.rpm
 308             echo "$(rpm -q $pkg) $(echo $url)"  >> /tmp/installedpackages.txt
 309         fi
 310     fi
 311 done < /tmp/rpms-wanted.txt
 312 ' > $PKG_SCRIPT
 313
 314 do_ssh "cat - > /tmp/installpackages.sh" < $PKG_SCRIPT
 315 do_ssh "cat - > /tmp/rpms-wanted.txt" < $RPMS_WANTED_FILE
 316 do_ssh chmod +x /tmp/installpackages.sh
 317 do_ssh /tmp/installpackages.sh
 318
 319 ###
 320 ### Extract package list with versions and urls
 321 ###
 322 sshpass -p "$SSH_PASS" scp -o StrictHostKeyChecking=false -o UserKnownHostsFile=/dev/null  $SSH_USER@${ip}:/tmp/installedpackages.txt $RPM_TEMPFILE
 323
 324 if [ -e ${REPO_MOD_FILE} ] ; then
 325     cp $REPO_MOD_FILE $REPO_OUTPUTFILE
 326 fi
 327 cat $RPM_TEMPFILE | sort > $RPM_OUTPUTFILE
 328 rm -f $RPM_TEMPFILE
 329
 330 ### Get Python data. We do this by installing as per our
 331 ### requirements.txt file while fetching a list of all
 332 ### installed modules before and after, and then comparing.
 333
 334 PIP_TEMPFILE_BEFORE=$(mktemp)
 335 PIP_TEMPFILE_AFTER=$(mktemp)
 336 do_ssh "cat - > /tmp/requirements.txt" < ${PIP_REQUIREMENTS}
 337 do_ssh pip list | sort > $PIP_TEMPFILE_BEFORE
 338 do_ssh pip install -r /tmp/requirements.txt
 339 do_ssh pip list | sort > $PIP_TEMPFILE_AFTER
 340
 341 comm -1 -3 ${PIP_TEMPFILE_BEFORE} ${PIP_TEMPFILE_AFTER} | \
 342   sed -e 's/\(.*\) (\(.*\))/\1==\2/' > $PIP_OUTPUTFILE
 343 rm -f $PIP_TEMPFILE_BEFORE
 344 rm -f $PIP_TEMPFILE_AFTER
 345
 346 ###
 347 ### Stop VIRL session
 348 ###
 349 virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} simengine-stop --session ${id}
 350 trap "" EXIT