resources/tools/disk-image-builder/centos/run-listmaker.sh

   1 #!/bin/bash
   2
   3 # Copyright (c) 2018 Cisco and/or its affiliates.
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at:
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 # This script is to spin up a simulation in VIRL, and fetch the URLs for all packages
  17 # that the user would obtain if they did an "yum update" today.
  18 #
  19 # This entire step is neither secure nor portable. The assumption --for now-- is that
  20 # this will only ever be run in LF CSIT VIRL lab. Should the requirement arise to
  21 # run this elsewhere, then additional work may be required to make this more
  22 # portable.
  23
  24 # This script requires that the following two environment variables be defined-
  25 #
  26 # $VIRL_USER
  27 # $VIRL_PASSWORD
  28
  29 VERSION=$(cat $(dirname $0)/CHANGELOG  | grep '^## ' | head -1 | sed -e 's/.*\[\(.*\)\].*/\1/')
  30 if [ "${VERSION}" = "" ]
  31 then
  32   echo "Unable to determine build version from CHANGELOG file. Make sure"
  33   echo "that there is an entry for the most recent version in CHANGELOG,"
  34   echo "and that the entry is formated like"
  35   echo
  36   echo "## [1.0] - 2016-05-20"
  37   exit 1
  38 fi
  39 DATE=$(date +%Y-%m-%d)
  40
  41 RPMS_TMP_DIR=`mktemp -d`
  42 RPMS_WANTED_FILE=$RPMS_TMP_DIR/rpms_wanted.txt
  43 REPO_MOD_FILE=$RPMS_TMP_DIR/Centos-Vault.repo
  44
  45 if [ "$1" == "centos-7.3-1611" ]
  46 then
  47     OS="centos-7.3-1611"
  48     VIRL_TOPOLOGY_FILE="listmaker/virl-listmaker-centos-7.3-1611.yaml"
  49 echo '
  50 # C7.3.1611
  51 [C7.3.1611-base]
  52 name=CentOS-7.3.1611 - Base
  53 baseurl=http://vault.centos.org/7.3.1611/os/$basearch/
  54 gpgcheck=1
  55 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
  56 enabled=1
  57
  58 [C7.3.1611-updates]
  59 name=CentOS-7.3.1611 - Updates
  60 baseurl=http://vault.centos.org/7.3.1611/updates/$basearch/
  61 gpgcheck=1
  62 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
  63 enabled=1
  64
  65 [C7.3.1611-extras]
  66 name=CentOS-7.3.1611 - Extras
  67 baseurl=http://vault.centos.org/7.3.1611/extras/$basearch/
  68 gpgcheck=1
  69 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
  70 enabled=1
  71
  72 [C7.3.1611-centosplus]
  73 name=CentOS-7.3.1611 - CentOSPlus
  74 baseurl=http://vault.centos.org/7.3.1611/centosplus/$basearch/
  75 gpgcheck=1
  76 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
  77 enabled=0
  78
  79 [C7.3.1611-fasttrack]
  80 name=CentOS-7.3.1611 - CentOSPlus
  81 baseurl=http://vault.centos.org/7.3.1611/fasttrack/$basearch/
  82 gpgcheck=1
  83 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
  84 enabled=0
  85
  86 ' >  $REPO_MOD_FILE
  87 else
  88     if [ "$1" == "centos-7.4-1711" ]
  89         then
  90             OS="centos-7.4-1711"
  91             VIRL_TOPOLOGY_FILE="listmaker/virl-listmaker-centos-7.4-1711.yaml"
  92     elif [ "$1" == "centos-7.6-1810" ]
  93         then
  94             OS="centos-7.6-1810"
  95             VIRL_TOPOLOGY_FILE="listmaker/virl-listmaker-centos-7.6-1810.yaml"
  96     else
  97         echo specify argument -- probably centos-7.3-1611 , centos-7.4-1711 or centos-7.6-1810
  98         exit 1
  99     fi
 100 fi
 101
 102 RELEASE="${OS}_${DATE}_${VERSION}"
 103 OUTPUT_DIR="lists/${RELEASE}"
 104
 105 echo "Building release ${RELEASE}."
 106 echo "Storing data in ${OUTPUT_DIR}/."
 107
 108
 109
 110 # RPM packages wanted
 111
 112 echo '
 113 #RPM_WANTLIST_INFRA
 114 nfs-utils
 115 cloud-init
 116 pkgconfig
 117 yum-utils
 118 #RPM_WANTLIST_CSIT
 119 python-devel
 120 python-virtualenv
 121 python-setuptools
 122 redhat-rpm-config
 123 epel-release
 124 python-srpm-macros
 125 python-rpm-macros
 126 python36-ply
 127 python36-devel
 128 python36-pip
 129 python2-pip-8.1.2-8.el7.noarch http://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/p/
 130 openssl-devel
 131 git
 132 trousers
 133 strongswan-5.7.2-1.el7.x86_64 http://mirror.math.princeton.edu/pub/epel/7/x86_64/Packages/s/
 134 python-cffi
 135 #RPM_WANTLIST_TLDK
 136 tcpdump
 137 #RPM_WANTLIST_VPP
 138 elfutils-libelf
 139 elfutils-libelf-devel
 140 kernel-debug-devel
 141 gcc
 142 dkms-2.6.1-1.el7.noarch https://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/d/
 143 bridge-utils
 144 selinux-policy
 145 selinux-policy-devel
 146 mbedtls-2.7.10-1.el7.x86_64 https://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/m/
 147 mbedtls-devel-2.7.10-1.el7.x86_64 https://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/m/
 148 #RPM_WANTLIST_TREX
 149 zlib-devel
 150 unzip
 151 #RPM_WANTLIST_MISC
 152 socat
 153 psmisc
 154 gperftools
 155 glusterfs
 156 glusterfs-api
 157 libiscsi
 158 libibverbs
 159 libpcap
 160 libpcap-devel
 161 pixman
 162 libpng
 163 pulseaudio-libs
 164 librados2
 165 librbd1
 166 librdmacm
 167 libseccomp
 168 spice-server
 169 spice-server-devel
 170 libusb
 171 usbredir
 172 glusterfs-devel
 173 seavgabios-bin
 174 sgabios-bin
 175 ipxe-roms-qemu
 176 nss-devel
 177 seabios-bin
 178 libffi-devel
 179 boost-filesystem
 180 #RPM_WANTLIST_NESTED
 181 trousers
 182 libnettle
 183 gnutls
 184 libcacard
 185 libcacard-tools
 186 libcacard-devel
 187 device-mapper-multipath-libs
 188 libepoxy
 189 libibumad
 190 qemu-img-ev-2.12.0-18.el7_6.3.1.x86_64 http://mirror.centos.org/centos/7/virt/x86_64/kvm-common/
 191 qemu-kvm-tools-ev-2.12.0-18.el7_6.3.1.x86_64 http://mirror.centos.org/centos/7/virt/x86_64/kvm-common/
 192 qemu-kvm-common-ev-2.12.0-18.el7_6.3.1.x86_64 http://mirror.centos.org/centos/7/virt/x86_64/kvm-common/
 193 qemu-kvm-ev-2.12.0-18.el7_6.3.1.x86_64 http://mirror.centos.org/centos/7/virt/x86_64/kvm-common/
 194 #RPM_WANTLIST_JAVA
 195 java-1.8.0-openjdk-headless
 196 java-1.8.0-openjdk-devel
 197 ' > $RPMS_WANTED_FILE
 198
 199 RPM_OUTPUTFILE="${OUTPUT_DIR}/rpm-packages.txt"
 200 REPO_OUTPUTFILE="${OUTPUT_DIR}/Centos-Vault.repo"
 201
 202 # Python requirements file. Can point to a manually crafted file
 203 # here, or to the actual CSIT requirements file, or to a symlink.
 204
 205 PIP_REQUIREMENTS="../../../../requirements.txt"
 206 if [ ! -f ${PIP_REQUIREMENTS} ]
 207 then
 208   echo "PIP requirements file ${PIP_REQUIREMENTS} not found."
 209   exit 1
 210 fi
 211
 212 PIP_OUTPUTFILE="${OUTPUT_DIR}/pip-requirements.txt"
 213
 214 # These will be used for SSH to the listmaker VM, and must match with what
 215 # was defined in the listmaker VM's kickstart file.
 216 SSH_USER="root"
 217 SSH_PASS="csit"
 218
 219 ###
 220 ### Spin up simulation
 221 ###
 222 if [ "$VIRL_USER" = "" ] || [ "$VIRL_PASSWORD" = "" ]
 223 then
 224   echo '$VIRL_USER and $VIRL_PASSWORD environment variables must be defined'
 225   exit 1
 226 fi
 227
 228 output=$(virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} \
 229   simengine-launch -f ${VIRL_TOPOLOGY_FILE} 2>&1)
 230 id=$(echo "${output}" | grep "Simulation ID is " | cut -f 4 -d ' ')
 231
 232 if [ "$id" = "" ]
 233 then
 234   echo "Did not get a simulation ID. Aborting."
 235   echo "Output was:"
 236   echo "${output}"
 237   exit 1
 238 fi
 239
 240 echo My ID is ${id}
 241 function stop_sim {
 242   virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} simengine-stop --session ${id}
 243 }
 244 trap stop_sim EXIT
 245
 246 ip="None"
 247 while [ "${ip}" = "None" ] || [ "${ip}" = "" ]
 248 do
 249   sleep 5
 250   output=$(virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} simengine-interfaces --session ${id} --nodes listmaker --interfaces management 2>&1)
 251   ip=$(echo "${output}" | grep "u'ip-address" | cut -f 4 -d "'" | cut -f 1 -d '/')
 252 done
 253 echo "IP is $ip"
 254
 255 sleep 10
 256
 257 if ping -w 60 -c 2 $ip > /dev/null
 258 then
 259   echo Host $ip alive
 260 else
 261   echo Host $ip failed to respond to ping
 262   exit 1
 263 fi
 264
 265 # Wait for SSH to be up
 266 while ! nc -z $ip 22
 267 do
 268   sleep 3
 269 done
 270
 271 if [ ! -d ${OUTPUT_DIR} ]; then
 272     mkdir -p $OUTPUT_DIR
 273 fi
 274
 275 ###
 276 ### SSH to the VM and perform package installation. Before each step,
 277 ### dry-run and grab the URLs of the packages that would be installed.
 278 ###
 279
 280 function do_ssh {
 281   # Helper function: SSH and avoid password prompt
 282   sshpass -p $SSH_PASS ssh -o StrictHostKeyChecking=false -o UserKnownHostsFile=/dev/null \
 283     -o LogLevel=error ${SSH_USER}@${ip} "$@"
 284 }
 285
 286 RPM_TEMPFILE=$(mktemp)
 287 ###
 288 ### If there is a repo file specified install it. Freeze yum to release specified above to
 289 ### avoid updating to be packages newer then the specified Centos release. Most packages are
 290 ### installed with yum from a specified Centos version. The packages with urls after them
 291 ### have specific versions and they are installed by rpm from the url.
 292 ###
 293
 294 tmp2=$(mktemp)
 295 echo '#!/bin/bash' > $tmp2
 296
 297 if [ -e ${REPO_MOD_FILE} ] ; then
 298     do_ssh cp /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.back
 299     do_ssh mv /etc/yum.repos.d/CentOS-Vault.repo /etc/yum.repos.d/CentOS-Vault.back
 300
 301     do_ssh "cat - > /tmp/tmp-Vault.repo" < ${REPO_MOD_FILE}
 302     do_ssh cp -f /tmp/tmp-Vault.repo /etc/yum.repos.d/CentOS-Vault.repo
 303
 304     echo "sed -i '/gpgcheck=1/s/.*/&\nenabled=0/' /etc/yum.repos.d/CentOS-Base.repo" >> $tmp2
 305     do_ssh "cat - > /tmp/chrepo.sh" < ${tmp2}
 306     do_ssh chmod +x /tmp/chrepo.sh
 307     do_ssh /tmp/chrepo.sh
 308 fi
 309 PKG_SCRIPT=$(mktemp)
 310 echo \
 311 'while IFS='' read -r line || [[ -n $line ]] ; do
 312     array=( $line )
 313     if [[ -z ${array[0]}  ]] ; then :;
 314     elif [[ ${array[0]:0:1} == "#" ]] ; then :;
 315     else
 316         pkg="${array[0]}"
 317         url="${array[1]}"
 318         if [[ -z $url ]] ; then
 319             yum install -y $pkg
 320             echo $pkg >> /tmp/installedpackages.txt
 321         else
 322             rpm -i --force $url$pkg.rpm
 323             echo "$(rpm -q $pkg) $(echo $url)"  >> /tmp/installedpackages.txt
 324         fi
 325     fi
 326 done < /tmp/rpms-wanted.txt
 327 ' > $PKG_SCRIPT
 328
 329 do_ssh "cat - > /tmp/installpackages.sh" < $PKG_SCRIPT
 330 do_ssh "cat - > /tmp/rpms-wanted.txt" < $RPMS_WANTED_FILE
 331 do_ssh chmod +x /tmp/installpackages.sh
 332 do_ssh /tmp/installpackages.sh
 333
 334 ###
 335 ### Extract package list with versions and urls
 336 ###
 337 sshpass -p "$SSH_PASS" scp -o StrictHostKeyChecking=false -o UserKnownHostsFile=/dev/null  $SSH_USER@${ip}:/tmp/installedpackages.txt $RPM_TEMPFILE
 338
 339 if [ -e ${REPO_MOD_FILE} ] ; then
 340     cp $REPO_MOD_FILE $REPO_OUTPUTFILE
 341 fi
 342 cat $RPM_TEMPFILE | sort > $RPM_OUTPUTFILE
 343 rm -f $RPM_TEMPFILE
 344
 345 ### Get Python data. We do this by installing as per our
 346 ### requirements.txt file while fetching a list of all
 347 ### installed modules before and after, and then comparing.
 348
 349 PIP_TEMPFILE_BEFORE=$(mktemp)
 350 PIP_TEMPFILE_AFTER=$(mktemp)
 351 do_ssh "cat - > /tmp/requirements.txt" < ${PIP_REQUIREMENTS}
 352 do_ssh pip list | sort > $PIP_TEMPFILE_BEFORE
 353 do_ssh pip install -r /tmp/requirements.txt
 354 do_ssh pip list | sort > $PIP_TEMPFILE_AFTER
 355
 356 comm -1 -3 ${PIP_TEMPFILE_BEFORE} ${PIP_TEMPFILE_AFTER} | \
 357   sed -e 's/\(.*\) (\(.*\))/\1==\2/' > $PIP_OUTPUTFILE
 358 rm -f $PIP_TEMPFILE_BEFORE
 359 rm -f $PIP_TEMPFILE_AFTER
 360
 361 ###
 362 ### Stop VIRL session
 363 ###
 364 virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} simengine-stop --session ${id}
 365 trap "" EXIT