3 # Copyright (c) 2018 Cisco and/or its affiliates.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
16 # This script is to spin up a simulation in VIRL, and fetch the URLs for all packages
17 # that the user would obtain if they did an "yum update" today.
19 # This entire step is neither secure nor portable. The assumption --for now-- is that
20 # this will only ever be run in LF CSIT VIRL lab. Should the requirement arise to
21 # run this elsewhere, then additional work may be required to make this more
24 # This script requires that the following two environment variables be defined-
29 VERSION=$(cat $(dirname $0)/CHANGELOG | grep '^## ' | head -1 | sed -e 's/.*\[\(.*\)\].*/\1/')
30 if [ "${VERSION}" = "" ]
32 echo "Unable to determine build version from CHANGELOG file. Make sure"
33 echo "that there is an entry for the most recent version in CHANGELOG,"
34 echo "and that the entry is formated like"
36 echo "## [1.0] - 2016-05-20"
39 DATE=$(date +%Y-%m-%d)
41 RPMS_TMP_DIR=`mktemp -d`
42 RPMS_WANTED_FILE=$RPMS_TMP_DIR/rpms_wanted.txt
43 REPO_MOD_FILE=$RPMS_TMP_DIR/Centos-Vault.repo
45 if [ "$1" == "centos-7.3-1611" ]
48 VIRL_TOPOLOGY_FILE="listmaker/virl-listmaker-centos-7.3-1611.yaml"
52 name=CentOS-7.3.1611 - Base
53 baseurl=http://vault.centos.org/7.3.1611/os/$basearch/
55 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
59 name=CentOS-7.3.1611 - Updates
60 baseurl=http://vault.centos.org/7.3.1611/updates/$basearch/
62 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
66 name=CentOS-7.3.1611 - Extras
67 baseurl=http://vault.centos.org/7.3.1611/extras/$basearch/
69 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
72 [C7.3.1611-centosplus]
73 name=CentOS-7.3.1611 - CentOSPlus
74 baseurl=http://vault.centos.org/7.3.1611/centosplus/$basearch/
76 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
80 name=CentOS-7.3.1611 - CentOSPlus
81 baseurl=http://vault.centos.org/7.3.1611/fasttrack/$basearch/
83 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
88 if [ "$1" == "centos-7.4-1711" ]
91 VIRL_TOPOLOGY_FILE="listmaker/virl-listmaker-centos-7.4-1711.yaml"
92 elif [ "$1" == "centos-7.6-1810" ]
95 VIRL_TOPOLOGY_FILE="listmaker/virl-listmaker-centos-7.6-1810.yaml"
97 echo specify argument -- probably centos-7.3-1611 , centos-7.4-1711 or centos-7.6-1810
102 RELEASE="${OS}_${DATE}_${VERSION}"
103 OUTPUT_DIR="lists/${RELEASE}"
105 echo "Building release ${RELEASE}."
106 echo "Storing data in ${OUTPUT_DIR}/."
110 # RPM packages wanted
129 python2-pip-8.1.2-8.el7.noarch http://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/p/
133 strongswan-5.7.2-1.el7.x86_64 http://mirror.math.princeton.edu/pub/epel/7/x86_64/Packages/s/
139 elfutils-libelf-devel
142 dkms-2.6.1-1.el7.noarch https://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/d/
146 mbedtls-2.7.10-1.el7.x86_64 https://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/m/
147 mbedtls-devel-2.7.10-1.el7.x86_64 https://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/m/
187 device-mapper-multipath-libs
190 qemu-img-ev-2.12.0-18.el7_6.3.1.x86_64 http://mirror.centos.org/centos/7/virt/x86_64/kvm-common/
191 qemu-kvm-tools-ev-2.12.0-18.el7_6.3.1.x86_64 http://mirror.centos.org/centos/7/virt/x86_64/kvm-common/
192 qemu-kvm-common-ev-2.12.0-18.el7_6.3.1.x86_64 http://mirror.centos.org/centos/7/virt/x86_64/kvm-common/
193 qemu-kvm-ev-2.12.0-18.el7_6.3.1.x86_64 http://mirror.centos.org/centos/7/virt/x86_64/kvm-common/
195 java-1.8.0-openjdk-headless
196 java-1.8.0-openjdk-devel
197 ' > $RPMS_WANTED_FILE
199 RPM_OUTPUTFILE="${OUTPUT_DIR}/rpm-packages.txt"
200 REPO_OUTPUTFILE="${OUTPUT_DIR}/Centos-Vault.repo"
202 # Python requirements file. Can point to a manually crafted file
203 # here, or to the actual CSIT requirements file, or to a symlink.
205 PIP_REQUIREMENTS="../../../../requirements.txt"
206 if [ ! -f ${PIP_REQUIREMENTS} ]
208 echo "PIP requirements file ${PIP_REQUIREMENTS} not found."
212 PIP_OUTPUTFILE="${OUTPUT_DIR}/pip-requirements.txt"
214 # These will be used for SSH to the listmaker VM, and must match with what
215 # was defined in the listmaker VM's kickstart file.
220 ### Spin up simulation
222 if [ "$VIRL_USER" = "" ] || [ "$VIRL_PASSWORD" = "" ]
224 echo '$VIRL_USER and $VIRL_PASSWORD environment variables must be defined'
228 output=$(virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} \
229 simengine-launch -f ${VIRL_TOPOLOGY_FILE} 2>&1)
230 id=$(echo "${output}" | grep "Simulation ID is " | cut -f 4 -d ' ')
234 echo "Did not get a simulation ID. Aborting."
242 virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} simengine-stop --session ${id}
247 while [ "${ip}" = "None" ] || [ "${ip}" = "" ]
250 output=$(virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} simengine-interfaces --session ${id} --nodes listmaker --interfaces management 2>&1)
251 ip=$(echo "${output}" | grep "u'ip-address" | cut -f 4 -d "'" | cut -f 1 -d '/')
257 if ping -w 60 -c 2 $ip > /dev/null
261 echo Host $ip failed to respond to ping
265 # Wait for SSH to be up
271 if [ ! -d ${OUTPUT_DIR} ]; then
276 ### SSH to the VM and perform package installation. Before each step,
277 ### dry-run and grab the URLs of the packages that would be installed.
281 # Helper function: SSH and avoid password prompt
282 sshpass -p $SSH_PASS ssh -o StrictHostKeyChecking=false -o UserKnownHostsFile=/dev/null \
283 -o LogLevel=error ${SSH_USER}@${ip} "$@"
286 RPM_TEMPFILE=$(mktemp)
288 ### If there is a repo file specified install it. Freeze yum to release specified above to
289 ### avoid updating to be packages newer then the specified Centos release. Most packages are
290 ### installed with yum from a specified Centos version. The packages with urls after them
291 ### have specific versions and they are installed by rpm from the url.
295 echo '#!/bin/bash' > $tmp2
297 if [ -e ${REPO_MOD_FILE} ] ; then
298 do_ssh cp /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.back
299 do_ssh mv /etc/yum.repos.d/CentOS-Vault.repo /etc/yum.repos.d/CentOS-Vault.back
301 do_ssh "cat - > /tmp/tmp-Vault.repo" < ${REPO_MOD_FILE}
302 do_ssh cp -f /tmp/tmp-Vault.repo /etc/yum.repos.d/CentOS-Vault.repo
304 echo "sed -i '/gpgcheck=1/s/.*/&\nenabled=0/' /etc/yum.repos.d/CentOS-Base.repo" >> $tmp2
305 do_ssh "cat - > /tmp/chrepo.sh" < ${tmp2}
306 do_ssh chmod +x /tmp/chrepo.sh
307 do_ssh /tmp/chrepo.sh
311 'while IFS='' read -r line || [[ -n $line ]] ; do
313 if [[ -z ${array[0]} ]] ; then :;
314 elif [[ ${array[0]:0:1} == "#" ]] ; then :;
318 if [[ -z $url ]] ; then
320 echo $pkg >> /tmp/installedpackages.txt
322 rpm -i --force $url$pkg.rpm
323 echo "$(rpm -q $pkg) $(echo $url)" >> /tmp/installedpackages.txt
326 done < /tmp/rpms-wanted.txt
329 do_ssh "cat - > /tmp/installpackages.sh" < $PKG_SCRIPT
330 do_ssh "cat - > /tmp/rpms-wanted.txt" < $RPMS_WANTED_FILE
331 do_ssh chmod +x /tmp/installpackages.sh
332 do_ssh /tmp/installpackages.sh
335 ### Extract package list with versions and urls
337 sshpass -p "$SSH_PASS" scp -o StrictHostKeyChecking=false -o UserKnownHostsFile=/dev/null $SSH_USER@${ip}:/tmp/installedpackages.txt $RPM_TEMPFILE
339 if [ -e ${REPO_MOD_FILE} ] ; then
340 cp $REPO_MOD_FILE $REPO_OUTPUTFILE
342 cat $RPM_TEMPFILE | sort > $RPM_OUTPUTFILE
345 ### Get Python data. We do this by installing as per our
346 ### requirements.txt file while fetching a list of all
347 ### installed modules before and after, and then comparing.
349 PIP_TEMPFILE_BEFORE=$(mktemp)
350 PIP_TEMPFILE_AFTER=$(mktemp)
351 do_ssh "cat - > /tmp/requirements.txt" < ${PIP_REQUIREMENTS}
352 do_ssh pip list | sort > $PIP_TEMPFILE_BEFORE
353 do_ssh pip install -r /tmp/requirements.txt
354 do_ssh pip list | sort > $PIP_TEMPFILE_AFTER
356 comm -1 -3 ${PIP_TEMPFILE_BEFORE} ${PIP_TEMPFILE_AFTER} | \
357 sed -e 's/\(.*\) (\(.*\))/\1==\2/' > $PIP_OUTPUTFILE
358 rm -f $PIP_TEMPFILE_BEFORE
359 rm -f $PIP_TEMPFILE_AFTER
362 ### Stop VIRL session
364 virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} simengine-stop --session ${id}