3 # Copyright (c) 2018 Cisco and/or its affiliates.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
16 # This script is to spin up a simulation in VIRL, and fetch the URLs for all packages
17 # that the user would obtain if they did an "yum update" today.
19 # This entire step is neither secure nor portable. The assumption --for now-- is that
20 # this will only ever be run in LF CSIT VIRL lab. Should the requirement arise to
21 # run this elsewhere, then additional work may be required to make this more
24 # This script requires that the following two environment variables be defined-
29 VERSION=$(cat $(dirname $0)/CHANGELOG | grep '^## ' | head -1 | sed -e 's/.*\[\(.*\)\].*/\1/')
30 if [ "${VERSION}" = "" ]
32 echo "Unable to determine build version from CHANGELOG file. Make sure"
33 echo "that there is an entry for the most recent version in CHANGELOG,"
34 echo "and that the entry is formated like"
36 echo "## [1.0] - 2016-05-20"
39 DATE=$(date +%Y-%m-%d)
41 RPMS_TMP_DIR=`mktemp -d`
42 RPMS_WANTED_FILE=$RPMS_TMP_DIR/rpms_wanted.txt
43 REPO_MOD_FILE=$RPMS_TMP_DIR/Centos-Vault.repo
45 if [ "$1" == "centos-7-1511" ]
48 VIRL_TOPOLOGY_FILE="listmaker/virl-listmaker-centos-7-1511.yaml"
50 if [ "$1" == "centos-7.3-1611" ]
53 VIRL_TOPOLOGY_FILE="listmaker/virl-listmaker-centos-7.3-1611.yaml"
57 name=CentOS-7.3.1611 - Base
58 baseurl=http://vault.centos.org/7.3.1611/os/$basearch/
60 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
64 name=CentOS-7.3.1611 - Updates
65 baseurl=http://vault.centos.org/7.3.1611/updates/$basearch/
67 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
71 name=CentOS-7.3.1611 - Extras
72 baseurl=http://vault.centos.org/7.3.1611/extras/$basearch/
74 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
77 [C7.3.1611-centosplus]
78 name=CentOS-7.3.1611 - CentOSPlus
79 baseurl=http://vault.centos.org/7.3.1611/centosplus/$basearch/
81 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
85 name=CentOS-7.3.1611 - CentOSPlus
86 baseurl=http://vault.centos.org/7.3.1611/fasttrack/$basearch/
88 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
92 elif [ "$1" == "centos-7.4-1711" ]
95 VIRL_TOPOLOGY_FILE="listmaker/virl-listmaker-centos-7.4-1711.yaml"
97 echo specify argument -- probably centos-7-1511 or centos-7.3-1611 or centos-7.4-1711
102 RELEASE="${OS}_${DATE}_${VERSION}"
103 OUTPUT_DIR="lists/${RELEASE}"
105 echo "Building release ${RELEASE}."
106 echo "Storing data in ${OUTPUT_DIR}/."
110 # RPM packages wanted
122 python2-pip-8.1.2-5.el7.noarch http://mirror.math.princeton.edu/pub/epel/7/x86_64/Packages/p/
125 strongswan-5.6.1-2.el7.x86_64 http://mirror.math.princeton.edu/pub/epel/7/x86_64/Packages/s/
131 elfutils-libelf-devel
134 dkms-2.4.0-1.20170926git959bd74.el7.noarch http://mirror.math.princeton.edu/pub/epel/7/x86_64/Packages/d/
138 mbedtls-2.7.0-1.el7.x86_64 http://mirror.math.princeton.edu/pub/epel/7/x86_64/Packages/m/
176 qemu-img-ev-2.9.0-16.el7_4.13.1.x86_64 http://mirror.centos.org/centos-7/7/virt/x86_64/kvm-common/
177 qemu-kvm-tools-ev-2.9.0-16.el7_4.13.1.x86_64 http://mirror.centos.org/centos-7/7/virt/x86_64/kvm-common/
178 qemu-kvm-common-ev-2.9.0-16.el7_4.13.1.x86_64 http://mirror.centos.org/centos-7/7/virt/x86_64/kvm-common/
179 qemu-kvm-ev-2.9.0-16.el7_4.13.1.x86_64 http://mirror.centos.org/centos-7/7/virt/x86_64/kvm-common/
181 java-1.8.0-openjdk-headless
182 java-1.8.0-openjdk-devel
183 ' > $RPMS_WANTED_FILE
185 RPM_OUTPUTFILE="${OUTPUT_DIR}/rpm-packages.txt"
186 REPO_OUTPUTFILE="${OUTPUT_DIR}/Centos-Vault.repo"
188 # Python requirements file. Can point to a manually crafted file
189 # here, or to the actual CSIT requirements file, or to a symlink.
191 PIP_REQUIREMENTS="../../../../requirements.txt"
192 if [ ! -f ${PIP_REQUIREMENTS} ]
194 echo "PIP requirements file ${PIP_REQUIREMENTS} not found."
198 PIP_OUTPUTFILE="${OUTPUT_DIR}/pip-requirements.txt"
200 # These will be used for SSH to the listmaker VM, and must match with what
201 # was defined in the listmaker VM's kickstart file.
206 ### Spin up simulation
208 if [ "$VIRL_USER" = "" ] || [ "$VIRL_PASSWORD" = "" ]
210 echo '$VIRL_USER and $VIRL_PASSWORD environment variables must be defined'
214 output=$(virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} \
215 simengine-launch -f ${VIRL_TOPOLOGY_FILE} 2>&1)
216 id=$(echo "${output}" | grep "Simulation ID is " | cut -f 4 -d ' ')
220 echo "Did not get a simulation ID. Aborting."
228 virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} simengine-stop --session ${id}
233 while [ "${ip}" = "None" ] || [ "${ip}" = "" ]
236 output=$(virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} simengine-interfaces --session ${id} --nodes listmaker --interfaces management 2>&1)
237 ip=$(echo "${output}" | grep "u'ip-address" | cut -f 4 -d "'" | cut -f 1 -d '/')
243 if ping -w 60 -c 2 $ip > /dev/null
247 echo Host $ip failed to respond to ping
251 # Wait for SSH to be up
257 if [ ! -d ${OUTPUT_DIR} ]; then
262 ### SSH to the VM and perform package installation. Before each step,
263 ### dry-run and grab the URLs of the packages that would be installed.
267 # Helper function: SSH and avoid password prompt
268 sshpass -p $SSH_PASS ssh -o StrictHostKeyChecking=false -o UserKnownHostsFile=/dev/null \
269 -o LogLevel=error ${SSH_USER}@${ip} "$@"
272 RPM_TEMPFILE=$(mktemp)
274 ### If there is a repo file specified install it. Freeze yum to release specified above to
275 ### avoid updating to be packages newer then the specified Centos release. Most packages are
276 ### installed with yum from a specified Centos version. The packages with urls after them
277 ### have specific versions and they are installed by rpm from the url.
281 echo '#!/bin/bash' > $tmp2
283 if [ -e ${REPO_MOD_FILE} ] ; then
284 do_ssh cp /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.back
285 do_ssh mv /etc/yum.repos.d/CentOS-Vault.repo /etc/yum.repos.d/CentOS-Vault.back
287 do_ssh "cat - > /tmp/tmp-Vault.repo" < ${REPO_MOD_FILE}
288 do_ssh cp -f /tmp/tmp-Vault.repo /etc/yum.repos.d/CentOS-Vault.repo
290 echo "sed -i '/gpgcheck=1/s/.*/&\nenabled=0/' /etc/yum.repos.d/CentOS-Base.repo" >> $tmp2
291 do_ssh "cat - > /tmp/chrepo.sh" < ${tmp2}
292 do_ssh chmod +x /tmp/chrepo.sh
293 do_ssh /tmp/chrepo.sh
297 'while IFS='' read -r line || [[ -n $line ]] ; do
299 if [[ -z ${array[0]} ]] ; then :;
300 elif [[ ${array[0]:0:1} == "#" ]] ; then :;
304 if [[ -z $url ]] ; then
306 echo $pkg >> /tmp/installedpackages.txt
308 rpm -i --force $url$pkg.rpm
309 echo "$(rpm -q $pkg) $(echo $url)" >> /tmp/installedpackages.txt
312 done < /tmp/rpms-wanted.txt
315 do_ssh "cat - > /tmp/installpackages.sh" < $PKG_SCRIPT
316 do_ssh "cat - > /tmp/rpms-wanted.txt" < $RPMS_WANTED_FILE
317 do_ssh chmod +x /tmp/installpackages.sh
318 do_ssh /tmp/installpackages.sh
321 ### Extract package list with versions and urls
323 sshpass -p "$SSH_PASS" scp -o StrictHostKeyChecking=false -o UserKnownHostsFile=/dev/null $SSH_USER@${ip}:/tmp/installedpackages.txt $RPM_TEMPFILE
325 if [ -e ${REPO_MOD_FILE} ] ; then
326 cp $REPO_MOD_FILE $REPO_OUTPUTFILE
328 cat $RPM_TEMPFILE | sort > $RPM_OUTPUTFILE
331 ### Get Python data. We do this by installing as per our
332 ### requirements.txt file while fetching a list of all
333 ### installed modules before and after, and then comparing.
335 PIP_TEMPFILE_BEFORE=$(mktemp)
336 PIP_TEMPFILE_AFTER=$(mktemp)
337 do_ssh "cat - > /tmp/requirements.txt" < ${PIP_REQUIREMENTS}
338 do_ssh pip list | sort > $PIP_TEMPFILE_BEFORE
339 do_ssh pip install -r /tmp/requirements.txt
340 do_ssh pip list | sort > $PIP_TEMPFILE_AFTER
342 comm -1 -3 ${PIP_TEMPFILE_BEFORE} ${PIP_TEMPFILE_AFTER} | \
343 sed -e 's/\(.*\) (\(.*\))/\1==\2/' > $PIP_OUTPUTFILE
344 rm -f $PIP_TEMPFILE_BEFORE
345 rm -f $PIP_TEMPFILE_AFTER
348 ### Stop VIRL session
350 virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} simengine-stop --session ${id}