resources/tools/disk-image-builder/centos/run-listmaker.sh

   1 #!/bin/bash
   2
   3 # Copyright (c) 2018 Cisco and/or its affiliates.
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at:
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 # This script is to spin up a simulation in VIRL, and fetch the URLs for all packages
  17 # that the user would obtain if they did an "yum update" today.
  18 #
  19 # This entire step is neither secure nor portable. The assumption --for now-- is that
  20 # this will only ever be run in LF CSIT VIRL lab. Should the requirement arise to
  21 # run this elsewhere, then additional work may be required to make this more
  22 # portable.
  23
  24 # This script requires that the following two environment variables be defined-
  25 #
  26 # $VIRL_USER
  27 # $VIRL_PASSWORD
  28
  29 VERSION=$(cat $(dirname $0)/CHANGELOG  | grep '^## ' | head -1 | sed -e 's/.*\[\(.*\)\].*/\1/')
  30 if [ "${VERSION}" = "" ]
  31 then
  32   echo "Unable to determine build version from CHANGELOG file. Make sure"
  33   echo "that there is an entry for the most recent version in CHANGELOG,"
  34   echo "and that the entry is formated like"
  35   echo
  36   echo "## [1.0] - 2016-05-20"
  37   exit 1
  38 fi
  39 DATE=$(date +%Y-%m-%d)
  40
  41 RPMS_TMP_DIR=`mktemp -d`
  42 RPMS_WANTED_FILE=$RPMS_TMP_DIR/rpms_wanted.txt
  43 REPO_MOD_FILE=$RPMS_TMP_DIR/Centos-Vault.repo
  44
  45 if [ "$1" == "centos-7-1511" ]
  46 then
  47     OS="centos-7-1511"
  48     VIRL_TOPOLOGY_FILE="listmaker/virl-listmaker-centos-7-1511.yaml"
  49 else
  50     if [ "$1" == "centos-7.3-1611" ]
  51     then
  52         OS="centos-7.3-1611"
  53         VIRL_TOPOLOGY_FILE="listmaker/virl-listmaker-centos-7.3-1611.yaml"
  54     echo '
  55 # C7.3.1611
  56 [C7.3.1611-base]
  57 name=CentOS-7.3.1611 - Base
  58 baseurl=http://vault.centos.org/7.3.1611/os/$basearch/
  59 gpgcheck=1
  60 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
  61 enabled=1
  62
  63 [C7.3.1611-updates]
  64 name=CentOS-7.3.1611 - Updates
  65 baseurl=http://vault.centos.org/7.3.1611/updates/$basearch/
  66 gpgcheck=1
  67 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
  68 enabled=1
  69
  70 [C7.3.1611-extras]
  71 name=CentOS-7.3.1611 - Extras
  72 baseurl=http://vault.centos.org/7.3.1611/extras/$basearch/
  73 gpgcheck=1
  74 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
  75 enabled=1
  76
  77 [C7.3.1611-centosplus]
  78 name=CentOS-7.3.1611 - CentOSPlus
  79 baseurl=http://vault.centos.org/7.3.1611/centosplus/$basearch/
  80 gpgcheck=1
  81 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
  82 enabled=0
  83
  84 [C7.3.1611-fasttrack]
  85 name=CentOS-7.3.1611 - CentOSPlus
  86 baseurl=http://vault.centos.org/7.3.1611/fasttrack/$basearch/
  87 gpgcheck=1
  88 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
  89 enabled=0
  90
  91 ' >  $REPO_MOD_FILE
  92     elif [ "$1" == "centos-7.4-1711" ]
  93     then
  94         OS="centos-7.4-1711"
  95         VIRL_TOPOLOGY_FILE="listmaker/virl-listmaker-centos-7.4-1711.yaml"
  96     else
  97         echo specify argument -- probably centos-7-1511 or centos-7.3-1611 or centos-7.4-1711
  98         exit 1
  99     fi
 100 fi
 101
 102 RELEASE="${OS}_${DATE}_${VERSION}"
 103 OUTPUT_DIR="lists/${RELEASE}"
 104
 105 echo "Building release ${RELEASE}."
 106 echo "Storing data in ${OUTPUT_DIR}/."
 107
 108
 109
 110 # RPM packages wanted
 111
 112 echo '
 113 #RPM_WANTLIST_INFRA
 114 nfs-utils
 115 cloud-init
 116 pkgconfig
 117 yum-utils
 118 #RPM_WANTLIST_CSIT
 119 python-devel
 120 python-virtualenv
 121 python-setuptools
 122 python2-pip-8.1.2-5.el7.noarch http://mirror.math.princeton.edu/pub/epel/7/x86_64/Packages/p/
 123 openssl-devel
 124 git
 125 strongswan-5.6.1-2.el7.x86_64 http://mirror.math.princeton.edu/pub/epel/7/x86_64/Packages/s/
 126 python-cffi
 127 #RPM_WANTLIST_TLDK
 128 tcpdump
 129 #RPM_WANTLIST_VPP
 130 elfutils-libelf
 131 elfutils-libelf-devel
 132 kernel-debug-devel
 133 gcc
 134 dkms-2.4.0-1.20170926git959bd74.el7.noarch http://mirror.math.princeton.edu/pub/epel/7/x86_64/Packages/d/
 135 bridge-utils
 136 selinux-policy
 137 selinux-policy-devel
 138 mbedtls-2.7.0-1.el7.x86_64 http://mirror.math.princeton.edu/pub/epel/7/x86_64/Packages/m/
 139 #RPM_WANTLIST_TREX
 140 zlib-devel
 141 unzip
 142 #RPM_WANTLIST_MISC
 143 socat
 144 psmisc
 145 gperftools
 146 glusterfs
 147 glusterfs-api
 148 libiscsi
 149 libibverbs
 150 libpcap
 151 libpcap-devel
 152 pixman
 153 libpng
 154 pulseaudio-libs
 155 librados2
 156 librbd1
 157 librdmacm
 158 libseccomp
 159 spice-server
 160 libusb
 161 usbredir
 162 glusterfs-devel
 163 seavgabios-bin
 164 sgabios-bin
 165 ipxe-roms-qemu
 166 nss-devel
 167 seabios-bin
 168 libffi-devel
 169 #RPM_WANTLIST_NESTED
 170 trousers
 171 libnettle
 172 gnutls
 173 libcacard
 174 libcacard-tools
 175 libcacard-devel
 176 qemu-img-ev-2.9.0-16.el7_4.13.1.x86_64 http://mirror.centos.org/centos-7/7/virt/x86_64/kvm-common/
 177 qemu-kvm-tools-ev-2.9.0-16.el7_4.13.1.x86_64 http://mirror.centos.org/centos-7/7/virt/x86_64/kvm-common/
 178 qemu-kvm-common-ev-2.9.0-16.el7_4.13.1.x86_64 http://mirror.centos.org/centos-7/7/virt/x86_64/kvm-common/
 179 qemu-kvm-ev-2.9.0-16.el7_4.13.1.x86_64 http://mirror.centos.org/centos-7/7/virt/x86_64/kvm-common/
 180 #RPM_WANTLIST_JAVA
 181 java-1.8.0-openjdk-headless
 182 java-1.8.0-openjdk-devel
 183 ' > $RPMS_WANTED_FILE
 184
 185 RPM_OUTPUTFILE="${OUTPUT_DIR}/rpm-packages.txt"
 186 REPO_OUTPUTFILE="${OUTPUT_DIR}/Centos-Vault.repo"
 187
 188 # Python requirements file. Can point to a manually crafted file
 189 # here, or to the actual CSIT requirements file, or to a symlink.
 190
 191 PIP_REQUIREMENTS="../../../../requirements.txt"
 192 if [ ! -f ${PIP_REQUIREMENTS} ]
 193 then
 194   echo "PIP requirements file ${PIP_REQUIREMENTS} not found."
 195   exit 1
 196 fi
 197
 198 PIP_OUTPUTFILE="${OUTPUT_DIR}/pip-requirements.txt"
 199
 200 # These will be used for SSH to the listmaker VM, and must match with what
 201 # was defined in the listmaker VM's kickstart file.
 202 SSH_USER="root"
 203 SSH_PASS="csit"
 204
 205 ###
 206 ### Spin up simulation
 207 ###
 208 if [ "$VIRL_USER" = "" ] || [ "$VIRL_PASSWORD" = "" ]
 209 then
 210   echo '$VIRL_USER and $VIRL_PASSWORD environment variables must be defined'
 211   exit 1
 212 fi
 213
 214 output=$(virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} \
 215   simengine-launch -f ${VIRL_TOPOLOGY_FILE} 2>&1)
 216 id=$(echo "${output}" | grep "Simulation ID is " | cut -f 4 -d ' ')
 217
 218 if [ "$id" = "" ]
 219 then
 220   echo "Did not get a simulation ID. Aborting."
 221   echo "Output was:"
 222   echo "${output}"
 223   exit 1
 224 fi
 225
 226 echo My ID is ${id}
 227 function stop_sim {
 228   virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} simengine-stop --session ${id}
 229 }
 230 trap stop_sim EXIT
 231
 232 ip="None"
 233 while [ "${ip}" = "None" ] || [ "${ip}" = "" ]
 234 do
 235   sleep 5
 236   output=$(virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} simengine-interfaces --session ${id} --nodes listmaker --interfaces management 2>&1)
 237   ip=$(echo "${output}" | grep "u'ip-address" | cut -f 4 -d "'" | cut -f 1 -d '/')
 238 done
 239 echo "IP is $ip"
 240
 241 sleep 10
 242
 243 if ping -w 60 -c 2 $ip > /dev/null
 244 then
 245   echo Host $ip alive
 246 else
 247   echo Host $ip failed to respond to ping
 248   exit 1
 249 fi
 250
 251 # Wait for SSH to be up
 252 while ! nc -z $ip 22
 253 do
 254   sleep 3
 255 done
 256
 257 if [ ! -d ${OUTPUT_DIR} ]; then
 258     mkdir -p $OUTPUT_DIR
 259 fi
 260
 261 ###
 262 ### SSH to the VM and perform package installation. Before each step,
 263 ### dry-run and grab the URLs of the packages that would be installed.
 264 ###
 265
 266 function do_ssh {
 267   # Helper function: SSH and avoid password prompt
 268   sshpass -p $SSH_PASS ssh -o StrictHostKeyChecking=false -o UserKnownHostsFile=/dev/null \
 269     -o LogLevel=error ${SSH_USER}@${ip} "$@"
 270 }
 271
 272 RPM_TEMPFILE=$(mktemp)
 273 ###
 274 ### If there is a repo file specified install it. Freeze yum to release specified above to
 275 ### avoid updating to be packages newer then the specified Centos release. Most packages are
 276 ### installed with yum from a specified Centos version. The packages with urls after them
 277 ### have specific versions and they are installed by rpm from the url.
 278 ###
 279
 280 tmp2=$(mktemp)
 281 echo '#!/bin/bash' > $tmp2
 282
 283 if [ -e ${REPO_MOD_FILE} ] ; then
 284     do_ssh cp /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.back
 285     do_ssh mv /etc/yum.repos.d/CentOS-Vault.repo /etc/yum.repos.d/CentOS-Vault.back
 286
 287     do_ssh "cat - > /tmp/tmp-Vault.repo" < ${REPO_MOD_FILE}
 288     do_ssh cp -f /tmp/tmp-Vault.repo /etc/yum.repos.d/CentOS-Vault.repo
 289
 290     echo "sed -i '/gpgcheck=1/s/.*/&\nenabled=0/' /etc/yum.repos.d/CentOS-Base.repo" >> $tmp2
 291     do_ssh "cat - > /tmp/chrepo.sh" < ${tmp2}
 292     do_ssh chmod +x /tmp/chrepo.sh
 293     do_ssh /tmp/chrepo.sh
 294 fi
 295 PKG_SCRIPT=$(mktemp)
 296 echo \
 297 'while IFS='' read -r line || [[ -n $line ]] ; do
 298     array=( $line )
 299     if [[ -z ${array[0]}  ]] ; then :;
 300     elif [[ ${array[0]:0:1} == "#" ]] ; then :;
 301     else
 302         pkg="${array[0]}"
 303         url="${array[1]}"
 304         if [[ -z $url ]] ; then
 305             yum install -y $pkg
 306             echo $pkg >> /tmp/installedpackages.txt
 307         else
 308             rpm -i --force $url$pkg.rpm
 309             echo "$(rpm -q $pkg) $(echo $url)"  >> /tmp/installedpackages.txt
 310         fi
 311     fi
 312 done < /tmp/rpms-wanted.txt
 313 ' > $PKG_SCRIPT
 314
 315 do_ssh "cat - > /tmp/installpackages.sh" < $PKG_SCRIPT
 316 do_ssh "cat - > /tmp/rpms-wanted.txt" < $RPMS_WANTED_FILE
 317 do_ssh chmod +x /tmp/installpackages.sh
 318 do_ssh /tmp/installpackages.sh
 319
 320 ###
 321 ### Extract package list with versions and urls
 322 ###
 323 sshpass -p "$SSH_PASS" scp -o StrictHostKeyChecking=false -o UserKnownHostsFile=/dev/null  $SSH_USER@${ip}:/tmp/installedpackages.txt $RPM_TEMPFILE
 324
 325 if [ -e ${REPO_MOD_FILE} ] ; then
 326     cp $REPO_MOD_FILE $REPO_OUTPUTFILE
 327 fi
 328 cat $RPM_TEMPFILE | sort > $RPM_OUTPUTFILE
 329 rm -f $RPM_TEMPFILE
 330
 331 ### Get Python data. We do this by installing as per our
 332 ### requirements.txt file while fetching a list of all
 333 ### installed modules before and after, and then comparing.
 334
 335 PIP_TEMPFILE_BEFORE=$(mktemp)
 336 PIP_TEMPFILE_AFTER=$(mktemp)
 337 do_ssh "cat - > /tmp/requirements.txt" < ${PIP_REQUIREMENTS}
 338 do_ssh pip list | sort > $PIP_TEMPFILE_BEFORE
 339 do_ssh pip install -r /tmp/requirements.txt
 340 do_ssh pip list | sort > $PIP_TEMPFILE_AFTER
 341
 342 comm -1 -3 ${PIP_TEMPFILE_BEFORE} ${PIP_TEMPFILE_AFTER} | \
 343   sed -e 's/\(.*\) (\(.*\))/\1==\2/' > $PIP_OUTPUTFILE
 344 rm -f $PIP_TEMPFILE_BEFORE
 345 rm -f $PIP_TEMPFILE_AFTER
 346
 347 ###
 348 ### Stop VIRL session
 349 ###
 350 virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} simengine-stop --session ${id}
 351 trap "" EXIT