3 # Copyright (c) 2018 Cisco and/or its affiliates.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
16 # This script is to spin up a simulation in VIRL, and fetch the URLs for all packages
17 # that the user would obtain if they did an "apt-get update", "apt-get upgrade" today.
19 # This entire step is neither secure nor portable. The assumption --for now-- is that
20 # this will only ever be run in LF CSIT VIRL lab. Should the requirement arise to
21 # run this elsewhere, then additional work may be required to make this more
24 # This script requires that the following two environment variables be defined-
29 VERSION=$(cat $(dirname $0)/CHANGELOG | grep '^## ' | head -1 | sed -e 's/.*\[\(.*\)\].*/\1/')
30 if [ "${VERSION}" = "" ]
32 echo "Unable to determine build version from CHANGELOG file. Make sure"
33 echo "that there is an entry for the most recent version in CHANGELOG,"
34 echo "and that the entry is formated like"
36 echo "## [1.0] - 2016-05-20"
39 DATE=$(date +%Y-%m-%d)
41 if [ "$1" = "ubuntu-14.04.4" ]
44 VIRL_TOPOLOGY_FILE="listmaker/virl-listmaker-ubuntu-14.04.4.yaml"
45 elif [ "$1" = "ubuntu-16.04.1" ]
48 VIRL_TOPOLOGY_FILE="listmaker/virl-listmaker-ubuntu-16.04.1.yaml"
50 echo "Please provide OS as parameter:"
51 echo "Options: ${0} [ubuntu-14.04.4|ubuntu-16.04.1]"
55 RELEASE="${OS}_${DATE}_${VERSION}"
56 OUTPUT_DIR="lists/${RELEASE}"
58 echo "Building release ${RELEASE}."
59 echo "Storing data in ${OUTPUT_DIR}/."
64 APT_WANTLIST_INFRA="nfs-common cloud-init"
65 APT_WANTLIST_CSIT="python-dev python-pip python-virtualenv git strongswan socat python-cffi python3-cffi"
66 APT_WANTLIST_TLDK="libpcap0.8-dev libpcap-dev cmake tcpdump"
67 APT_WANTLIST_VPP="dkms bridge-utils libmbedcrypto0 libmbedcrypto1 libmbedtls10 libmbedx509-0"
68 APT_WANTLIST_TREX="zlib1g-dev unzip"
69 APT_WANTLIST_NESTED="qemu-system-x86"
70 APT_WANTLIST_JAVA="openjdk-8-jdk-headless"
71 #Docker is currently disabled due to issues with apt repositories retrieval
72 #APT_WANTLIST_DOCKER="docker-engine"
74 # For now, let us NOT incude WANTLIST_NESTED in the below. We're installing qemu
75 # separately from a separate source.
76 APT_WANTLIST="$APT_WANTLIST_INFRA $APT_WANTLIST_CSIT $APT_WANTLIST_VPP $APT_WANTLIST_TREX $APT_WANTLIST_TLDK"
78 APT_OUTPUTFILE="${OUTPUT_DIR}/apt-packages.txt"
80 # Python requirements file. Can point to a manually crafted file
81 # here, or to the actual CSIT requirements file, or to a symlink.
83 PIP_REQUIREMENTS="../../../../requirements.txt"
84 if [ ! -f ${PIP_REQUIREMENTS} ]
86 echo "PIP requirements file ${PIP_REQUIREMENTS} not found."
90 PIP_OUTPUTFILE="${OUTPUT_DIR}/pip-requirements.txt"
92 # These will be used for SSH to the listmaker VM, and must match with what
93 # was defined in the listmaker VM's kickstart file.
98 ### Spin up simulation
100 if [ "$VIRL_USER" = "" ] || [ "$VIRL_PASSWORD" = "" ]
102 echo '$VIRL_USER and $VIRL_PASSWORD environment variables must be defined'
106 output=$(virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} \
107 simengine-launch -f ${VIRL_TOPOLOGY_FILE} 2>&1)
108 id=$(echo "${output}" | grep "Simulation ID is " | cut -f 4 -d ' ')
112 echo "Did not get a simulation ID. Aborting."
120 virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} simengine-stop --session ${id}
125 while [ "${ip}" = "None" ] || [ "${ip}" = "" ]
128 output=$(virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} simengine-interfaces --session ${id} --nodes listmaker --interfaces management 2>&1)
129 ip=$(echo "${output}" | grep "u'ip-address" | cut -f 4 -d "'" | cut -f 1 -d '/')
135 if ping -w 60 -c 10 $ip > /dev/null
139 echo Host $ip failed to respond to ping
143 # Wait for SSH to be up
152 ### SSH to the VM and perform package installation. Before each step,
153 ### dry-run and grab the URLs of the packages that would be installed.
157 # Helper function: SSH and avoid password prompt
158 sshpass -p $SSH_PASS ssh -o StrictHostKeyChecking=false -o UserKnownHostsFile=/dev/null \
159 -o LogLevel=error ${SSH_USER}@${ip} "$@"
162 if [ "$OS" = "ubuntu-14.04.4" ]
164 do_ssh "cat - > /etc/apt/sources.list" <<_EOF
165 deb http://us.archive.ubuntu.com/ubuntu/ trusty main restricted
166 deb-src http://us.archive.ubuntu.com/ubuntu/ trusty main restricted
167 deb http://us.archive.ubuntu.com/ubuntu/ trusty-updates main restricted
168 deb-src http://us.archive.ubuntu.com/ubuntu/ trusty-updates main restricted
169 deb http://us.archive.ubuntu.com/ubuntu/ trusty universe
170 deb-src http://us.archive.ubuntu.com/ubuntu/ trusty universe
171 deb http://us.archive.ubuntu.com/ubuntu/ trusty-updates universe
172 deb-src http://us.archive.ubuntu.com/ubuntu/ trusty-updates universe
173 deb http://us.archive.ubuntu.com/ubuntu/ trusty multiverse
174 deb-src http://us.archive.ubuntu.com/ubuntu/ trusty multiverse
175 deb http://us.archive.ubuntu.com/ubuntu/ trusty-updates multiverse
176 deb-src http://us.archive.ubuntu.com/ubuntu/ trusty-updates multiverse
177 deb http://us.archive.ubuntu.com/ubuntu/ trusty-backports main restricted universe multiverse
178 deb-src http://us.archive.ubuntu.com/ubuntu/ trusty-backports main restricted universe multiverse
179 deb http://security.ubuntu.com/ubuntu trusty-security main restricted
180 deb-src http://security.ubuntu.com/ubuntu trusty-security main restricted
181 deb http://security.ubuntu.com/ubuntu trusty-security universe
182 deb-src http://security.ubuntu.com/ubuntu trusty-security universe
183 deb http://security.ubuntu.com/ubuntu trusty-security multiverse
184 deb-src http://security.ubuntu.com/ubuntu trusty-security multiverse
186 elif [ "$OS" = "ubuntu-16.04.1" ]
188 do_ssh "cat - > /etc/apt/sources.list" <<_EOF
189 deb http://us.archive.ubuntu.com/ubuntu/ xenial main restricted
190 deb-src http://us.archive.ubuntu.com/ubuntu/ xenial main restricted
191 deb http://us.archive.ubuntu.com/ubuntu/ xenial-updates main restricted
192 deb-src http://us.archive.ubuntu.com/ubuntu/ xenial-updates main restricted
193 deb http://us.archive.ubuntu.com/ubuntu/ xenial universe
194 deb-src http://us.archive.ubuntu.com/ubuntu/ xenial universe
195 deb http://us.archive.ubuntu.com/ubuntu/ xenial-updates universe
196 deb-src http://us.archive.ubuntu.com/ubuntu/ xenial-updates universe
197 deb http://us.archive.ubuntu.com/ubuntu/ xenial multiverse
198 deb-src http://us.archive.ubuntu.com/ubuntu/ xenial multiverse
199 deb http://us.archive.ubuntu.com/ubuntu/ xenial-updates multiverse
200 deb-src http://us.archive.ubuntu.com/ubuntu/ xenial-updates multiverse
201 deb http://us.archive.ubuntu.com/ubuntu/ xenial-backports main restricted universe multiverse
202 deb-src http://us.archive.ubuntu.com/ubuntu/ xenial-backports main restricted universe multiverse
203 deb http://security.ubuntu.com/ubuntu xenial-security main restricted
204 deb-src http://security.ubuntu.com/ubuntu xenial-security main restricted
205 deb http://security.ubuntu.com/ubuntu xenial-security universe
206 deb-src http://security.ubuntu.com/ubuntu xenial-security universe
207 deb http://security.ubuntu.com/ubuntu xenial-security multiverse
208 deb-src http://security.ubuntu.com/ubuntu xenial-security multiverse
212 ### FIXME: Need error handling around all this
213 do_ssh apt-get update
215 APT_TEMPFILE=$(mktemp)
216 do_ssh apt-get --print-uris -y dist-upgrade >> $APT_TEMPFILE
217 do_ssh DEBIAN_FRONTEND=noninteractive apt-get -y dist-upgrade
218 do_ssh apt-get --print-uris -y install $APT_WANTLIST >> $APT_TEMPFILE
219 do_ssh DEBIAN_FRONTEND=noninteractive apt-get -y install $APT_WANTLIST
221 ### Install qemu ($APT_WANTLIST_NESTED) separately from PPA
222 if [ "$OS" = "ubuntu-14.04.4" ]
224 do_ssh "cat - >> /etc/apt/sources.list" <<_EOF
225 # For a custom qemu build
226 deb http://ppa.launchpad.net/syseleven-platform/virtualization/ubuntu trusty main
227 deb-src http://ppa.launchpad.net/syseleven-platform/virtualization/ubuntu trusty main
230 do_ssh apt-get --allow-unauthenticated update
231 do_ssh apt-get --print-uris --allow-unauthenticated -y install $APT_WANTLIST_NESTED >> $APT_TEMPFILE
232 do_ssh DEBIAN_FRONTEND=noninteractive apt-get --allow-unauthenticated -y install $APT_WANTLIST_NESTED
234 ### Install Java ($APT_WANTLIST_JAVA) separately from PPA
235 if [ "$OS" = "ubuntu-14.04.4" ]
237 do_ssh "cat - >> /etc/apt/sources.list" <<_EOF
239 deb http://ppa.launchpad.net/openjdk-r/ppa/ubuntu trusty main
242 do_ssh apt-get --allow-unauthenticated update
243 do_ssh apt-get --print-uris --allow-unauthenticated -y install $APT_WANTLIST_JAVA >> $APT_TEMPFILE
244 do_ssh DEBIAN_FRONTEND=noninteractive apt-get --allow-unauthenticated -y install $APT_WANTLIST_JAVA
246 ### Install Docker ($APT_WANTLIST_DOCKER) separately from PPA
247 #if [ "$OS" = "ubuntu-14.04.4" ]
249 #do_ssh "cat - >> /etc/apt/sources.list" <<_EOF
251 #deb https://apt.dockerproject.org/repo ubuntu-trusty main
253 #elif [ "$OS" = "ubuntu-16.04.1" ]
255 #do_ssh "cat - >> /etc/apt/sources.list" <<_EOF
257 #deb https://apt.dockerproject.org/repo ubuntu-xenial main
260 #do_ssh apt-get --allow-unauthenticated update
261 #do_ssh apt-get --print-uris --allow-unauthenticated -y install $APT_WANTLIST_DOCKER >> $APT_TEMPFILE
262 #do_ssh DEBIAN_FRONTEND=noninteractive apt-get --allow-unauthenticated -y install $APT_WANTLIST_DOCKER
264 cat $APT_TEMPFILE | grep MD5Sum | sort > $APT_OUTPUTFILE
267 ### Get Python data. We do this by installing as per our
268 ### requirements.txt file while fetching a list of all
269 ### installed modules before and after, and then comparing.
271 PIP_TEMPFILE_BEFORE=$(mktemp)
272 PIP_TEMPFILE_AFTER=$(mktemp)
273 do_ssh "cat - > /tmp/requirements.txt" < ${PIP_REQUIREMENTS}
274 do_ssh pip list | sort > $PIP_TEMPFILE_BEFORE
275 do_ssh pip install -r /tmp/requirements.txt
276 do_ssh pip list | sort > $PIP_TEMPFILE_AFTER
278 comm -1 -3 ${PIP_TEMPFILE_BEFORE} ${PIP_TEMPFILE_AFTER} | \
279 sed -e 's/\(.*\) (\(.*\))/\1==\2/' > $PIP_OUTPUTFILE
280 rm -f $PIP_TEMPFILE_BEFORE
281 rm -f $PIP_TEMPFILE_AFTER
284 ### Stop VIRL session
286 virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} simengine-stop --session ${id}