3 # Copyright (c) 2016 Cisco and/or its affiliates.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
16 # This script is to spin up a simulation in VIRL, and fetch the URLs for all packages
17 # that the user would obtain if they did an "apt-get update", "apt-get upgrade" today.
19 # This entire step is neither secure nor portable. The assumption --for now-- is that
20 # this will only ever be run in LF CSIT VIRL lab. Should the requirement arise to
21 # run this elsewhere, then additional work may be required to make this more
24 # This script requires that the following two environment variables be defined-
29 VERSION=$(cat $(dirname $0)/CHANGELOG | grep '^## ' | head -1 | sed -e 's/.*\[\(.*\)\].*/\1/')
30 if [ "${VERSION}" = "" ]
32 echo "Unable to determine build version from CHANGELOG file. Make sure"
33 echo "that there is an entry for the most recent version in CHANGELOG,"
34 echo "and that the entry is formated like"
36 echo "## [1.0] - 2016-05-20"
39 DATE=$(date +%Y-%m-%d)
41 if [ "$1" == "ubuntu-14.04.4" ]
44 VIRL_TOPOLOGY_FILE="listmaker/virl-listmaker-ubuntu-14.04.4.yaml"
45 elif [ "$1" == "ubuntu-16.04.1" ]
48 VIRL_TOPOLOGY_FILE="listmaker/virl-listmaker-ubuntu-16.04.1.yaml"
50 echo "Please provide OS as parameter:"
51 echo "Options: ${0} [ubuntu-14.04.4|ubuntu-16.04.1]"
55 RELEASE="${OS}_${DATE}_${VERSION}"
56 OUTPUT_DIR="lists/${RELEASE}"
58 echo "Building release ${RELEASE}."
59 echo "Storing data in ${OUTPUT_DIR}/."
64 APT_WANTLIST_INFRA="nfs-common cloud-init"
65 APT_WANTLIST_CSIT="python-dev python-pip python-virtualenv git strongswan"
66 APT_WANTLIST_VPP="dkms bridge-utils"
67 APT_WANTLIST_TREX="zlib1g-dev unzip"
68 APT_WANTLIST_NESTED="qemu-system-x86"
69 APT_WANTLIST_JAVA="openjdk-8-jdk-headless"
70 #Docker is currently disabled due to issues with apt repositories retrieval
71 #APT_WANTLIST_DOCKER="docker-engine"
73 # For now, let us NOT incude WANTLIST_NESTED in the below. We're installing qemu
74 # separately from a separate source.
75 APT_WANTLIST="$APT_WANTLIST_INFRA $APT_WANTLIST_CSIT $APT_WANTLIST_VPP $WANTLIST_TREX"
77 APT_OUTPUTFILE="${OUTPUT_DIR}/apt-packages.txt"
79 # Python requirements file. Can point to a manually crafted file
80 # here, or to the actual CSIT requirements file, or to a symlink.
82 PIP_REQUIREMENTS="../../../../requirements.txt"
83 if [ ! -f ${PIP_REQUIREMENTS} ]
85 echo "PIP requirements file ${PIP_REQUIREMENTS} not found."
89 PIP_OUTPUTFILE="${OUTPUT_DIR}/pip-requirements.txt"
91 # These will be used for SSH to the listmaker VM, and must match with what
92 # was defined in the listmaker VM's kickstart file.
97 ### Spin up simulation
99 if [ "$VIRL_USER" = "" ] || [ "$VIRL_PASSWORD" = "" ]
101 echo '$VIRL_USER and $VIRL_PASSWORD environment variables must be defined'
105 output=$(virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} \
106 simengine-launch -f ${VIRL_TOPOLOGY_FILE} 2>&1)
107 id=$(echo "${output}" | grep "Simulation ID is " | cut -f 4 -d ' ')
111 echo "Did not get a simulation ID. Aborting."
119 virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} simengine-stop --session ${id}
124 while [ "${ip}" = "None" ] || [ "${ip}" = "" ]
127 output=$(virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} simengine-interfaces --session ${id} --nodes listmaker --interfaces management 2>&1)
128 ip=$(echo "${output}" | grep "u'ip-address" | cut -f 4 -d "'" | cut -f 1 -d '/')
134 if ping -w 60 -c 2 $ip > /dev/null
138 echo Host $ip failed to respond to ping
142 # Wait for SSH to be up
151 ### SSH to the VM and perform package installation. Before each step,
152 ### dry-run and grab the URLs of the packages that would be installed.
156 # Helper function: SSH and avoid password prompt
157 sshpass -p $SSH_PASS ssh -o StrictHostKeyChecking=false -o UserKnownHostsFile=/dev/null \
158 -o LogLevel=error ${SSH_USER}@${ip} "$@"
161 if [ "$OS" = "ubuntu-14.04.4" ]
163 do_ssh "cat - > /etc/apt/sources.list" <<_EOF
164 deb http://us.archive.ubuntu.com/ubuntu/ trusty main restricted
165 deb-src http://us.archive.ubuntu.com/ubuntu/ trusty main restricted
166 deb http://us.archive.ubuntu.com/ubuntu/ trusty-updates main restricted
167 deb-src http://us.archive.ubuntu.com/ubuntu/ trusty-updates main restricted
168 deb http://us.archive.ubuntu.com/ubuntu/ trusty universe
169 deb-src http://us.archive.ubuntu.com/ubuntu/ trusty universe
170 deb http://us.archive.ubuntu.com/ubuntu/ trusty-updates universe
171 deb-src http://us.archive.ubuntu.com/ubuntu/ trusty-updates universe
172 deb http://us.archive.ubuntu.com/ubuntu/ trusty multiverse
173 deb-src http://us.archive.ubuntu.com/ubuntu/ trusty multiverse
174 deb http://us.archive.ubuntu.com/ubuntu/ trusty-updates multiverse
175 deb-src http://us.archive.ubuntu.com/ubuntu/ trusty-updates multiverse
176 deb http://us.archive.ubuntu.com/ubuntu/ trusty-backports main restricted universe multiverse
177 deb-src http://us.archive.ubuntu.com/ubuntu/ trusty-backports main restricted universe multiverse
178 deb http://security.ubuntu.com/ubuntu trusty-security main restricted
179 deb-src http://security.ubuntu.com/ubuntu trusty-security main restricted
180 deb http://security.ubuntu.com/ubuntu trusty-security universe
181 deb-src http://security.ubuntu.com/ubuntu trusty-security universe
182 deb http://security.ubuntu.com/ubuntu trusty-security multiverse
183 deb-src http://security.ubuntu.com/ubuntu trusty-security multiverse
185 elif [ "$OS" = "ubuntu-16.04.1" ]
187 do_ssh "cat - > /etc/apt/sources.list" <<_EOF
188 deb http://us.archive.ubuntu.com/ubuntu/ xenial main restricted
189 deb-src http://us.archive.ubuntu.com/ubuntu/ xenial main restricted
190 deb http://us.archive.ubuntu.com/ubuntu/ xenial-updates main restricted
191 deb-src http://us.archive.ubuntu.com/ubuntu/ xenial-updates main restricted
192 deb http://us.archive.ubuntu.com/ubuntu/ xenial universe
193 deb-src http://us.archive.ubuntu.com/ubuntu/ xenial universe
194 deb http://us.archive.ubuntu.com/ubuntu/ xenial-updates universe
195 deb-src http://us.archive.ubuntu.com/ubuntu/ xenial-updates universe
196 deb http://us.archive.ubuntu.com/ubuntu/ xenial multiverse
197 deb-src http://us.archive.ubuntu.com/ubuntu/ xenial multiverse
198 deb http://us.archive.ubuntu.com/ubuntu/ xenial-updates multiverse
199 deb-src http://us.archive.ubuntu.com/ubuntu/ xenial-updates multiverse
200 deb http://us.archive.ubuntu.com/ubuntu/ xenial-backports main restricted universe multiverse
201 deb-src http://us.archive.ubuntu.com/ubuntu/ xenial-backports main restricted universe multiverse
202 deb http://security.ubuntu.com/ubuntu xenial-security main restricted
203 deb-src http://security.ubuntu.com/ubuntu xenial-security main restricted
204 deb http://security.ubuntu.com/ubuntu xenial-security universe
205 deb-src http://security.ubuntu.com/ubuntu xenial-security universe
206 deb http://security.ubuntu.com/ubuntu xenial-security multiverse
207 deb-src http://security.ubuntu.com/ubuntu xenial-security multiverse
211 ### FIXME: Need error handling around all this
212 do_ssh apt-get update
214 APT_TEMPFILE=$(mktemp)
215 do_ssh apt-get --print-uris -y dist-upgrade >> $APT_TEMPFILE
216 do_ssh DEBIAN_FRONTEND=noninteractive apt-get -y dist-upgrade
217 do_ssh apt-get --print-uris -y install $APT_WANTLIST >> $APT_TEMPFILE
218 do_ssh DEBIAN_FRONTEND=noninteractive apt-get -y install $APT_WANTLIST
220 ### Install qemu ($APT_WANTLIST_NESTED) separately from PPA
221 if [ "$OS" = "ubuntu-14.04.4" ]
223 do_ssh "cat - >> /etc/apt/sources.list" <<_EOF
224 # For a custom qemu build
225 deb http://ppa.launchpad.net/syseleven-platform/virtualization/ubuntu trusty main
226 deb-src http://ppa.launchpad.net/syseleven-platform/virtualization/ubuntu trusty main
229 do_ssh apt-get --allow-unauthenticated update
230 do_ssh apt-get --print-uris --allow-unauthenticated -y install $APT_WANTLIST_NESTED >> $APT_TEMPFILE
231 do_ssh DEBIAN_FRONTEND=noninteractive apt-get --allow-unauthenticated -y install $APT_WANTLIST_NESTED
233 ### Install Java ($APT_WANTLIST_JAVA) separately from PPA
234 if [ "$OS" = "ubuntu-14.04.4" ]
236 do_ssh "cat - >> /etc/apt/sources.list" <<_EOF
238 deb http://ppa.launchpad.net/openjdk-r/ppa/ubuntu trusty main
241 do_ssh apt-get --allow-unauthenticated update
242 do_ssh apt-get --print-uris --allow-unauthenticated -y install $APT_WANTLIST_JAVA >> $APT_TEMPFILE
243 do_ssh DEBIAN_FRONTEND=noninteractive apt-get --allow-unauthenticated -y install $APT_WANTLIST_JAVA
245 ### Install Docker ($APT_WANTLIST_DOCKER) separately from PPA
246 #if [ "$OS" = "ubuntu-14.04.4" ]
248 #do_ssh "cat - >> /etc/apt/sources.list" <<_EOF
250 #deb https://apt.dockerproject.org/repo ubuntu-trusty main
252 #elif [ "$OS" = "ubuntu-16.04.1" ]
254 #do_ssh "cat - >> /etc/apt/sources.list" <<_EOF
256 #deb https://apt.dockerproject.org/repo ubuntu-xenial main
259 #do_ssh apt-get --allow-unauthenticated update
260 #do_ssh apt-get --print-uris --allow-unauthenticated -y install $APT_WANTLIST_DOCKER >> $APT_TEMPFILE
261 #do_ssh DEBIAN_FRONTEND=noninteractive apt-get --allow-unauthenticated -y install $APT_WANTLIST_DOCKER
263 cat $APT_TEMPFILE | grep MD5Sum | sort > $APT_OUTPUTFILE
266 ### Get Python data. We do this by installing as per our
267 ### requirements.txt file while fetching a list of all
268 ### installed modules before and after, and then comparing.
270 PIP_TEMPFILE_BEFORE=$(mktemp)
271 PIP_TEMPFILE_AFTER=$(mktemp)
272 do_ssh "cat - > /tmp/requirements.txt" < ${PIP_REQUIREMENTS}
273 do_ssh pip list | sort > $PIP_TEMPFILE_BEFORE
274 do_ssh pip install -r /tmp/requirements.txt
275 do_ssh pip list | sort > $PIP_TEMPFILE_AFTER
277 comm -1 -3 ${PIP_TEMPFILE_BEFORE} ${PIP_TEMPFILE_AFTER} | \
278 sed -e 's/\(.*\) (\(.*\))/\1==\2/' > $PIP_OUTPUTFILE
279 rm -f $PIP_TEMPFILE_BEFORE
280 rm -f $PIP_TEMPFILE_AFTER
283 ### Stop VIRL session
285 virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} simengine-stop --session ${id}