3 # Copyright (c) 2018 Cisco and/or its affiliates.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
16 # This script is to spin up a simulation in VIRL, and fetch the URLs for all packages
17 # that the user would obtain if they did an "apt-get update", "apt-get upgrade" today.
19 # This entire step is neither secure nor portable. The assumption --for now-- is that
20 # this will only ever be run in LF CSIT VIRL lab. Should the requirement arise to
21 # run this elsewhere, then additional work may be required to make this more
24 # This script requires that the following two environment variables be defined-
29 VERSION=$(cat $(dirname $0)/CHANGELOG | grep '^## ' | head -1 | sed -e 's/.*\[\(.*\)\].*/\1/')
30 if [ "${VERSION}" = "" ]
32 echo "Unable to determine build version from CHANGELOG file. Make sure"
33 echo "that there is an entry for the most recent version in CHANGELOG,"
34 echo "and that the entry is formated like"
36 echo "## [1.0] - 2016-05-20"
39 DATE=$(date +%Y-%m-%d)
41 if [ "$1" = "ubuntu-14.04.4" ]
44 VIRL_TOPOLOGY_FILE="listmaker/virl-listmaker-ubuntu-14.04.4.yaml"
45 elif [ "$1" = "ubuntu-16.04.1" ]
48 VIRL_TOPOLOGY_FILE="listmaker/virl-listmaker-ubuntu-16.04.1.yaml"
50 echo "Please provide OS as parameter:"
51 echo "Options: ${0} [ubuntu-14.04.4|ubuntu-16.04.1]"
55 RELEASE="${OS}_${DATE}_${VERSION}"
56 OUTPUT_DIR="lists/${RELEASE}"
58 echo "Building release ${RELEASE}."
59 echo "Storing data in ${OUTPUT_DIR}/."
64 APT_WANTLIST_INFRA="nfs-common cloud-init"
65 APT_WANTLIST_CSIT="python python-minimal python-dev python-pip python-virtualenv
66 python2.7 python2.7-minimal git strongswan socat python-cffi python3-cffi
67 python3 python3-cffi python3-cffi-backend python3-ply python3-pycparser
69 APT_WANTLIST_TLDK="libpcap0.8-dev libpcap-dev cmake tcpdump"
70 APT_WANTLIST_VPP="dkms bridge-utils libmbedcrypto0 libmbedtls10 libmbedx509-0
71 libpython2.7-minimal libpython-stdlib libpython2.7-stdlib libc6
72 python-pycparser python-ply libssl1.0"
73 APT_WANTLIST_TREX="zlib1g-dev unzip"
74 APT_WANTLIST_NESTED="qemu-system-x86"
75 APT_WANTLIST_JAVA="openjdk-8-jdk-headless"
76 #Docker is currently disabled due to issues with apt repositories retrieval
77 #APT_WANTLIST_DOCKER="docker-engine"
79 # For now, let us NOT incude WANTLIST_NESTED in the below. We're installing qemu
80 # separately from a separate source.
81 APT_WANTLIST="$APT_WANTLIST_INFRA $APT_WANTLIST_CSIT $APT_WANTLIST_VPP $APT_WANTLIST_TREX $APT_WANTLIST_TLDK"
83 APT_OUTPUTFILE="${OUTPUT_DIR}/apt-packages.txt"
85 # Python requirements file. Can point to a manually crafted file
86 # here, or to the actual CSIT requirements file, or to a symlink.
88 PIP_REQUIREMENTS="../../../../requirements.txt"
89 if [ ! -f ${PIP_REQUIREMENTS} ]
91 echo "PIP requirements file ${PIP_REQUIREMENTS} not found."
95 PIP_OUTPUTFILE="${OUTPUT_DIR}/pip-requirements.txt"
97 # These will be used for SSH to the listmaker VM, and must match with what
98 # was defined in the listmaker VM's kickstart file.
103 ### Spin up simulation
105 if [ "$VIRL_USER" = "" ] || [ "$VIRL_PASSWORD" = "" ]
107 echo '$VIRL_USER and $VIRL_PASSWORD environment variables must be defined'
111 output=$(virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} \
112 simengine-launch -f ${VIRL_TOPOLOGY_FILE} 2>&1)
113 id=$(echo "${output}" | grep "Simulation ID is " | cut -f 4 -d ' ')
117 echo "Did not get a simulation ID. Aborting."
125 virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} simengine-stop --session ${id}
130 while [ "${ip}" = "None" ] || [ "${ip}" = "" ]
133 output=$(virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} simengine-interfaces --session ${id} --nodes listmaker --interfaces management 2>&1)
134 ip=$(echo "${output}" | grep "u'ip-address" | cut -f 4 -d "'" | cut -f 1 -d '/')
140 if ping -w 60 -c 10 $ip > /dev/null
144 echo Host $ip failed to respond to ping
148 # Wait for SSH to be up
157 ### SSH to the VM and perform package installation. Before each step,
158 ### dry-run and grab the URLs of the packages that would be installed.
162 # Helper function: SSH and avoid password prompt
163 sshpass -p $SSH_PASS ssh -o StrictHostKeyChecking=false -o UserKnownHostsFile=/dev/null \
164 -o LogLevel=error ${SSH_USER}@${ip} "$@"
167 if [ "$OS" = "ubuntu-14.04.4" ]
169 do_ssh "cat - > /etc/apt/sources.list" <<_EOF
170 deb http://us.archive.ubuntu.com/ubuntu/ trusty main restricted
171 deb-src http://us.archive.ubuntu.com/ubuntu/ trusty main restricted
172 deb http://us.archive.ubuntu.com/ubuntu/ trusty-updates main restricted
173 deb-src http://us.archive.ubuntu.com/ubuntu/ trusty-updates main restricted
174 deb http://us.archive.ubuntu.com/ubuntu/ trusty universe
175 deb-src http://us.archive.ubuntu.com/ubuntu/ trusty universe
176 deb http://us.archive.ubuntu.com/ubuntu/ trusty-updates universe
177 deb-src http://us.archive.ubuntu.com/ubuntu/ trusty-updates universe
178 deb http://us.archive.ubuntu.com/ubuntu/ trusty multiverse
179 deb-src http://us.archive.ubuntu.com/ubuntu/ trusty multiverse
180 deb http://us.archive.ubuntu.com/ubuntu/ trusty-updates multiverse
181 deb-src http://us.archive.ubuntu.com/ubuntu/ trusty-updates multiverse
182 deb http://us.archive.ubuntu.com/ubuntu/ trusty-backports main restricted universe multiverse
183 deb-src http://us.archive.ubuntu.com/ubuntu/ trusty-backports main restricted universe multiverse
184 deb http://security.ubuntu.com/ubuntu trusty-security main restricted
185 deb-src http://security.ubuntu.com/ubuntu trusty-security main restricted
186 deb http://security.ubuntu.com/ubuntu trusty-security universe
187 deb-src http://security.ubuntu.com/ubuntu trusty-security universe
188 deb http://security.ubuntu.com/ubuntu trusty-security multiverse
189 deb-src http://security.ubuntu.com/ubuntu trusty-security multiverse
191 elif [ "$OS" = "ubuntu-16.04.1" ]
193 do_ssh "cat - > /etc/apt/sources.list" <<_EOF
194 deb http://us.archive.ubuntu.com/ubuntu/ xenial main restricted
195 deb-src http://us.archive.ubuntu.com/ubuntu/ xenial main restricted
196 deb http://us.archive.ubuntu.com/ubuntu/ xenial-updates main restricted
197 deb-src http://us.archive.ubuntu.com/ubuntu/ xenial-updates main restricted
198 deb http://us.archive.ubuntu.com/ubuntu/ xenial universe
199 deb-src http://us.archive.ubuntu.com/ubuntu/ xenial universe
200 deb http://us.archive.ubuntu.com/ubuntu/ xenial-updates universe
201 deb-src http://us.archive.ubuntu.com/ubuntu/ xenial-updates universe
202 deb http://us.archive.ubuntu.com/ubuntu/ xenial multiverse
203 deb-src http://us.archive.ubuntu.com/ubuntu/ xenial multiverse
204 deb http://us.archive.ubuntu.com/ubuntu/ xenial-updates multiverse
205 deb-src http://us.archive.ubuntu.com/ubuntu/ xenial-updates multiverse
206 deb http://us.archive.ubuntu.com/ubuntu/ xenial-backports main restricted universe multiverse
207 deb-src http://us.archive.ubuntu.com/ubuntu/ xenial-backports main restricted universe multiverse
208 deb http://security.ubuntu.com/ubuntu xenial-security main restricted
209 deb-src http://security.ubuntu.com/ubuntu xenial-security main restricted
210 deb http://security.ubuntu.com/ubuntu xenial-security universe
211 deb-src http://security.ubuntu.com/ubuntu xenial-security universe
212 deb http://security.ubuntu.com/ubuntu xenial-security multiverse
213 deb-src http://security.ubuntu.com/ubuntu xenial-security multiverse
217 ### FIXME: Need error handling around all this
218 do_ssh apt-get update
220 APT_TEMPFILE=$(mktemp)
221 do_ssh apt-get --print-uris -y dist-upgrade >> $APT_TEMPFILE
222 do_ssh DEBIAN_FRONTEND=noninteractive apt-get -y dist-upgrade
223 do_ssh apt-get --print-uris -y install $APT_WANTLIST >> $APT_TEMPFILE
224 do_ssh DEBIAN_FRONTEND=noninteractive apt-get -y install $APT_WANTLIST
226 ### Install qemu ($APT_WANTLIST_NESTED) separately from PPA
227 if [ "$OS" = "ubuntu-14.04.4" ]
229 do_ssh "cat - >> /etc/apt/sources.list" <<_EOF
230 # For a custom qemu build
231 deb http://ppa.launchpad.net/syseleven-platform/virtualization/ubuntu trusty main
232 deb-src http://ppa.launchpad.net/syseleven-platform/virtualization/ubuntu trusty main
235 do_ssh apt-get --allow-unauthenticated update
236 do_ssh apt-get --print-uris --allow-unauthenticated -y install $APT_WANTLIST_NESTED >> $APT_TEMPFILE
237 do_ssh DEBIAN_FRONTEND=noninteractive apt-get --allow-unauthenticated -y install $APT_WANTLIST_NESTED
239 ### Install Java ($APT_WANTLIST_JAVA) separately from PPA
240 if [ "$OS" = "ubuntu-14.04.4" ]
242 do_ssh "cat - >> /etc/apt/sources.list" <<_EOF
244 deb http://ppa.launchpad.net/openjdk-r/ppa/ubuntu trusty main
247 do_ssh apt-get --allow-unauthenticated update
248 do_ssh apt-get --print-uris --allow-unauthenticated -y install $APT_WANTLIST_JAVA >> $APT_TEMPFILE
249 do_ssh DEBIAN_FRONTEND=noninteractive apt-get --allow-unauthenticated -y install $APT_WANTLIST_JAVA
251 ### Install Docker ($APT_WANTLIST_DOCKER) separately from PPA
252 #if [ "$OS" = "ubuntu-14.04.4" ]
254 #do_ssh "cat - >> /etc/apt/sources.list" <<_EOF
256 #deb https://apt.dockerproject.org/repo ubuntu-trusty main
258 #elif [ "$OS" = "ubuntu-16.04.1" ]
260 #do_ssh "cat - >> /etc/apt/sources.list" <<_EOF
262 #deb https://apt.dockerproject.org/repo ubuntu-xenial main
265 #do_ssh apt-get --allow-unauthenticated update
266 #do_ssh apt-get --print-uris --allow-unauthenticated -y install $APT_WANTLIST_DOCKER >> $APT_TEMPFILE
267 #do_ssh DEBIAN_FRONTEND=noninteractive apt-get --allow-unauthenticated -y install $APT_WANTLIST_DOCKER
269 cat $APT_TEMPFILE | grep MD5Sum | sort > $APT_OUTPUTFILE
272 ### Get Python data. We do this by installing as per our
273 ### requirements.txt file while fetching a list of all
274 ### installed modules before and after, and then comparing.
276 PIP_TEMPFILE_BEFORE=$(mktemp)
277 PIP_TEMPFILE_AFTER=$(mktemp)
278 do_ssh "cat - > /tmp/requirements.txt" < ${PIP_REQUIREMENTS}
279 do_ssh pip list | sort > $PIP_TEMPFILE_BEFORE
280 do_ssh pip install -r /tmp/requirements.txt
281 do_ssh pip list | sort > $PIP_TEMPFILE_AFTER
283 comm -1 -3 ${PIP_TEMPFILE_BEFORE} ${PIP_TEMPFILE_AFTER} | \
284 sed -e 's/\(.*\) (\(.*\))/\1==\2/' > $PIP_OUTPUTFILE
285 rm -f $PIP_TEMPFILE_BEFORE
286 rm -f $PIP_TEMPFILE_AFTER
289 ### Stop VIRL session
291 virl_std_client -u ${VIRL_USER} -p ${VIRL_PASSWORD} simengine-stop --session ${id}