3 # Copyright (c) 2018 Cisco and/or its affiliates.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
16 # This script is to spin up a simulation in VIRL, and fetch the URLs for all packages
17 # that the user would obtain if they did an "yum update" today.
19 # This entire step is neither secure nor portable. The assumption --for now-- is that
20 # this will only ever be run in LF CSIT VIRL lab. Should the requirement arise to
21 # run this elsewhere, then additional work may be required to make this more
24 # This script requires that the following two environment variables be defined-
29 VERSION=$(cat $(dirname $0)/CHANGELOG | grep '^## ' | head -1 | sed -e 's/.*\[\(.*\)\].*/\1/')
30 if [ "${VERSION}" = "" ]
32 echo "Unable to determine build version from CHANGELOG file. Make sure"
33 echo "that there is an entry for the most recent version in CHANGELOG,"
34 echo "and that the entry is formated like"
36 echo "## [1.0] - 2016-05-20"
39 DATE=$(date +%Y-%m-%d)
41 RPMS_TMP_DIR=`mktemp -d`
42 RPMS_WANTED_FILE=$RPMS_TMP_DIR/rpms_wanted.txt
43 REPO_MOD_FILE=$RPMS_TMP_DIR/Centos-Vault.repo
45 if [ "$1" == "centos-7.3-1611" ]
48 VIRL_TOPOLOGY_FILE="listmaker/virl-listmaker-centos-7.3-1611.yaml"
52 name=CentOS-7.3.1611 - Base
53 baseurl=http://vault.centos.org/7.3.1611/os/$basearch/
55 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
59 name=CentOS-7.3.1611 - Updates
60 baseurl=http://vault.centos.org/7.3.1611/updates/$basearch/
62 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
66 name=CentOS-7.3.1611 - Extras
67 baseurl=http://vault.centos.org/7.3.1611/extras/$basearch/
69 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
72 [C7.3.1611-centosplus]
73 name=CentOS-7.3.1611 - CentOSPlus
74 baseurl=http://vault.centos.org/7.3.1611/centosplus/$basearch/
76 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
80 name=CentOS-7.3.1611 - CentOSPlus
81 baseurl=http://vault.centos.org/7.3.1611/fasttrack/$basearch/
83 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
88 if [ "$1" == "centos-7.4-1711" ]
91 VIRL_TOPOLOGY_FILE="listmaker/virl-listmaker-centos-7.4-1711.yaml"
92 elif [ "$1" == "centos-7.6-1810" ]
95 VIRL_TOPOLOGY_FILE="listmaker/virl-listmaker-centos-7.6-1810.yaml"
97 echo specify argument -- probably centos-7.3-1611 , centos-7.4-1711 or centos-7.6-1810
102 RELEASE="${OS}_${DATE}_${VERSION}"
103 OUTPUT_DIR="lists/${RELEASE}"
105 echo "Building release ${RELEASE}."
106 echo "Storing data in ${OUTPUT_DIR}/."
110 # RPM packages wanted
130 python2-pip-8.1.2-8.el7.noarch http://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/p/
134 strongswan-5.7.2-1.el7.x86_64 http://mirror.math.princeton.edu/pub/epel/7/x86_64/Packages/s/
140 elfutils-libelf-devel
143 dkms-2.6.1-1.el7.noarch https://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/d/
147 mbedtls-2.7.10-1.el7.x86_64 https://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/m/
148 mbedtls-devel-2.7.10-1.el7.x86_64 https://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/m/
188 device-mapper-multipath-libs
191 qemu-img-ev-2.12.0-18.el7_6.3.1.x86_64 http://mirror.centos.org/centos/7/virt/x86_64/kvm-common/
192 qemu-kvm-tools-ev-2.12.0-18.el7_6.3.1.x86_64 http://mirror.centos.org/centos/7/virt/x86_64/kvm-common/
193 qemu-kvm-common-ev-2.12.0-18.el7_6.3.1.x86_64 http://mirror.centos.org/centos/7/virt/x86_64/kvm-common/
194 qemu-kvm-ev-2.12.0-18.el7_6.3.1.x86_64 http://mirror.centos.org/centos/7/virt/x86_64/kvm-common/
196 java-1.8.0-openjdk-headless
197 java-1.8.0-openjdk-devel
198 ' > $RPMS_WANTED_FILE
200 RPM_OUTPUTFILE="${OUTPUT_DIR}/rpm-packages.txt"
201 REPO_OUTPUTFILE="${OUTPUT_DIR}/Centos-Vault.repo"
203 # Python requirements file. Can point to a manually crafted file
204 # here, or to the actual CSIT requirements file, or to a symlink.
206 PIP_REQUIREMENTS="../../../../requirements.txt"
207 if [ ! -f ${PIP_REQUIREMENTS} ]
209 echo "PIP requirements file ${PIP_REQUIREMENTS} not found."
213 PIP_OUTPUTFILE="${OUTPUT_DIR}/pip-requirements.txt"
215 # These will be used for SSH to the listmaker VM, and must match with what
216 # was defined in the listmaker VM's kickstart file.
221 ### Spin up simulation
223 if [ "$VIRL_USER" = "" ] || [ "$VIRL_PASSWORD" = "" ]
225 echo '$VIRL_USER and $VIRL_PASSWORD environment variables must be defined'
229 output=$(virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} \
230 simengine-launch -f ${VIRL_TOPOLOGY_FILE} 2>&1)
231 id=$(echo "${output}" | grep "Simulation ID is " | cut -f 4 -d ' ')
235 echo "Did not get a simulation ID. Aborting."
243 virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} simengine-stop --session ${id}
248 while [ "${ip}" = "None" ] || [ "${ip}" = "" ]
251 output=$(virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} simengine-interfaces --session ${id} --nodes listmaker --interfaces management 2>&1)
252 ip=$(echo "${output}" | grep "u'ip-address" | cut -f 4 -d "'" | cut -f 1 -d '/')
258 if ping -w 60 -c 2 $ip > /dev/null
262 echo Host $ip failed to respond to ping
266 # Wait for SSH to be up
272 if [ ! -d ${OUTPUT_DIR} ]; then
277 ### SSH to the VM and perform package installation. Before each step,
278 ### dry-run and grab the URLs of the packages that would be installed.
282 # Helper function: SSH and avoid password prompt
283 sshpass -p $SSH_PASS ssh -o StrictHostKeyChecking=false -o UserKnownHostsFile=/dev/null \
284 -o LogLevel=error ${SSH_USER}@${ip} "$@"
287 RPM_TEMPFILE=$(mktemp)
289 ### If there is a repo file specified install it. Freeze yum to release specified above to
290 ### avoid updating to be packages newer then the specified Centos release. Most packages are
291 ### installed with yum from a specified Centos version. The packages with urls after them
292 ### have specific versions and they are installed by rpm from the url.
296 echo '#!/bin/bash' > $tmp2
298 if [ -e ${REPO_MOD_FILE} ] ; then
299 do_ssh cp /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.back
300 do_ssh mv /etc/yum.repos.d/CentOS-Vault.repo /etc/yum.repos.d/CentOS-Vault.back
302 do_ssh "cat - > /tmp/tmp-Vault.repo" < ${REPO_MOD_FILE}
303 do_ssh cp -f /tmp/tmp-Vault.repo /etc/yum.repos.d/CentOS-Vault.repo
305 echo "sed -i '/gpgcheck=1/s/.*/&\nenabled=0/' /etc/yum.repos.d/CentOS-Base.repo" >> $tmp2
306 do_ssh "cat - > /tmp/chrepo.sh" < ${tmp2}
307 do_ssh chmod +x /tmp/chrepo.sh
308 do_ssh /tmp/chrepo.sh
312 'while IFS='' read -r line || [[ -n $line ]] ; do
314 if [[ -z ${array[0]} ]] ; then :;
315 elif [[ ${array[0]:0:1} == "#" ]] ; then :;
319 if [[ -z $url ]] ; then
321 echo $pkg >> /tmp/installedpackages.txt
323 rpm -i --force $url$pkg.rpm
324 echo "$(rpm -q $pkg) $(echo $url)" >> /tmp/installedpackages.txt
327 done < /tmp/rpms-wanted.txt
330 do_ssh "cat - > /tmp/installpackages.sh" < $PKG_SCRIPT
331 do_ssh "cat - > /tmp/rpms-wanted.txt" < $RPMS_WANTED_FILE
332 do_ssh chmod +x /tmp/installpackages.sh
333 do_ssh /tmp/installpackages.sh
336 ### Extract package list with versions and urls
338 sshpass -p "$SSH_PASS" scp -o StrictHostKeyChecking=false -o UserKnownHostsFile=/dev/null $SSH_USER@${ip}:/tmp/installedpackages.txt $RPM_TEMPFILE
340 if [ -e ${REPO_MOD_FILE} ] ; then
341 cp $REPO_MOD_FILE $REPO_OUTPUTFILE
343 cat $RPM_TEMPFILE | sort > $RPM_OUTPUTFILE
346 ### Get Python data. We do this by installing as per our
347 ### requirements.txt file while fetching a list of all
348 ### installed modules before and after, and then comparing.
350 PIP_TEMPFILE_BEFORE=$(mktemp)
351 PIP_TEMPFILE_AFTER=$(mktemp)
352 do_ssh "cat - > /tmp/requirements.txt" < ${PIP_REQUIREMENTS}
353 do_ssh pip list | sort > $PIP_TEMPFILE_BEFORE
354 do_ssh pip install -r /tmp/requirements.txt
355 do_ssh pip list | sort > $PIP_TEMPFILE_AFTER
357 comm -1 -3 ${PIP_TEMPFILE_BEFORE} ${PIP_TEMPFILE_AFTER} | \
358 sed -e 's/\(.*\) (\(.*\))/\1==\2/' > $PIP_OUTPUTFILE
359 rm -f $PIP_TEMPFILE_BEFORE
360 rm -f $PIP_TEMPFILE_AFTER
363 ### Stop VIRL session
365 virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} simengine-stop --session ${id}