1 # Copyright (c) 2019 Cisco and/or its affiliates.
2 # Copyright (c) 2019 PANTHEON.tech and/or its affiliates.
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at:
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
17 # This library defines functions used mainly by "bootstrap" entry scripts.
18 # Generally, the functions assume "common.sh" library has been sourced already.
20 # Keep functions ordered alphabetically, please.
22 # TODO: Add a link to bash style guide.
25 function gather_build () {
28 # - TEST_CODE - String affecting test selection, usually jenkins job name.
29 # - DOWNLOAD_DIR - Path to directory pybot takes the build to test from.
31 # - DUT - CSIT test/ subdirectory containing suites to execute.
32 # Directories updated:
33 # - ${DOWNLOAD_DIR} - Files needed by tests are gathered here.
35 # - die - Print to stderr and exit, defined in common.sh
36 # - gather_os - Parse os parameter for OS/distro name.
37 # - gather_dpdk, gather_vpp, gather_ligato - See their definitions.
38 # Multiple other side effects are possible,
39 # see functions called from here for their current description.
41 # TODO: Separate DUT-from-TEST_CODE from gather-for-DUT,
42 # when the first one becomes relevant for per_patch.
46 pushd "${DOWNLOAD_DIR}" || die "Pushd failed."
47 case "${TEST_CODE}" in
50 # FIXME: Avoid failing on empty ${DOWNLOAD_DIR}.
54 gather_vpp || die "The function should have died on error."
58 gather_ligato || die "The function should have died on error."
62 gather_dpdk || die "The function should have died on error."
65 die "Unable to identify DUT type from: ${TEST_CODE}"
68 popd || die "Popd failed."
72 function gather_dpdk () {
74 # Ensure latest DPDK archive is downloaded.
77 # - TEST_CODE - The test selection string from environment or argument.
79 # - dpdk archive name to download if TEST_CODE is not time based.
80 # Directories updated:
81 # - ./ - Assumed ${DOWNLOAD_DIR}, dpdk-*.tar.xz is downloaded if not there.
83 # - die - Print to stderr and exit, defined in common.sh
87 dpdk_repo="https://fast.dpdk.org/rel"
88 # Use downloaded packages with specific version
89 if [[ "${TEST_CODE}" == *"daily"* ]] || \
90 [[ "${TEST_CODE}" == *"weekly"* ]] || \
91 [[ "${TEST_CODE}" == *"timed"* ]];
93 echo "Downloading latest DPDK packages from repo..."
94 # URL is not in quotes, calling command from variable keeps them.
95 wget_command=("wget" "--no-check-certificate" "-nv" "-O" "-")
96 wget_command+=("${dpdk_repo}")
97 dpdk_stable_ver="$("${wget_command[@]}" | grep -v "2015"\
98 | grep -Eo 'dpdk-[^\"]+xz' | tail -1)" || {
99 die "Composite piped command failed."
102 echo "Downloading DPDK package of specific version from repo ..."
103 # Downloading DPDK version based on what VPP is using. Currently
104 # it is not easy way to detect from VPP version automatically.
105 dpdk_stable_ver="$(< "${CSIT_DIR}/DPDK_VPP_VER")".tar.xz || {
106 die "Failed to read DPDK VPP version!"
109 # TODO: Use "wget -N" instead checking for file presence?
110 if [[ ! -f "${dpdk_stable_ver}" ]]; then
111 wget -nv --no-check-certificate "${dpdk_repo}/${dpdk_stable_ver}" || {
112 die "Failed to get DPDK package from: ${dpdk_repo}"
118 function gather_ligato () {
120 # Build docker image (with vpp, ligato and vpp-agent),
121 # and put it to ${DOWNLOAD_DIR}/.
123 # Access rights needed for:
124 # - "wget", "git clone", "dpdk -x", "cd" above ${CSIT_DIR}.
125 # - "sudo" without password.
127 # - "dpdk -i" is allowed.
128 # - "docker" commands have everything they needs.
130 # - DOWNLOAD_DIR - Path to directory pybot takes the build to test from.
131 # - CSIT_DIR - Path to existing root of local CSIT git repository.
133 # - ${CSIT_DIR}/VPP_AGENT_STABLE_VER - Vpp agent version to use.
134 # Directories updated:
135 # - ${DOWNLOAD_DIR} - Docker image stored, VPP *.deb stored and deleted.
136 # - /tmp/vpp - VPP is unpacked there, not cleaned afterwards.
137 # - ${CSIT_DIR}/vpp-agent - Created, vpp-agent git repo si cloned there.
138 # - Also, various temporary files are stored there.
139 # System consequences:
140 # - Docker package is installed.
141 # - Presumably dockerd process is started.
142 # - The ligato/dev-vpp-agent docker image is downloaded.
143 # - Results of subsequent image manipulation are probably left lingering.
144 # Other hardcoded values:
145 # - Docker .deb file name to download and install.
147 # - die - Print to stderr and exit, defined in common_functions.sh
148 # - gather_vpp - See eponymous fragment file assumend to be sourced already.
149 # TODO: What is the best order of description items?
151 # TODO: Many of the following comments act as abstraction.
152 # But the abstracted blocks are mostly one-liners (plus "|| die"),
153 # so maybe it is not worth introducing fragments/functions for the blocks.
154 # TODO: This fragment is too long anyway, split it up.
158 gather_vpp || die "The function should have died on error."
160 mkdir -p /tmp/vpp && rm -f /tmp/vpp/* || {
161 die "Failed to create temporary directory!"
163 dpkg -x "${DOWNLOAD_DIR}/vpp_"*".deb" "/tmp/vpp" || {
164 die "Failed to extract VPP packages for kubernetes!"
167 ligato_repo_url="https://github.com/ligato/"
168 vpp_agent_stable_ver="$(< "${CSIT_DIR}/VPP_AGENT_STABLE_VER")" || {
169 die "Failed to read vpp-agent stable version!"
172 # Clone & checkout stable vpp-agent.
173 cd "${CSIT_DIR}" || die "Change directory failed!"
174 git clone -b master --single-branch \
175 "${ligato_repo_url}/vpp-agent" "vpp-agent" || {
176 die "Failed to run: git clone ${ligato_repo_url}/vpp-agent!"
178 cd "vpp-agent" || die "Change directory failed!"
181 curl -fsSL https://get.docker.com | sudo bash || {
182 die "Failed to install Docker package!"
185 # Pull ligato/dev_vpp_agent docker image and re-tag as local.
186 sudo docker pull "ligato/dev-vpp-agent:${vpp_agent_stable_ver}" || {
187 die "Failed to pull Docker image!"
189 params=(ligato/dev-vpp-agent:${vpp_agent_stable_ver} dev_vpp_agent:latest)
190 sudo docker tag "${params[@]}" || {
191 die "Failed to tag Docker image!"
194 # Start dev_vpp_agent container as daemon.
195 sudo docker run --rm -itd --name "agentcnt" "dev_vpp_agent" bash || {
196 die "Failed to run Docker image!"
199 # Copy latest vpp api into running container.
200 sudo docker exec agentcnt rm -rf "agentcnt:/usr/share/vpp/api" || {
201 die "Failed to remove previous API!"
203 sudo docker cp "/tmp/vpp/usr/share/vpp/api" "agentcnt:/usr/share/vpp" || {
204 die "Failed to copy files Docker image!"
207 # Recompile vpp-agent.
208 script_arg=". ~/.bashrc; cd /go/src/github.com/ligato/vpp-agent"
209 script_arg+=" && make generate && make install"
210 sudo docker exec -i agentcnt script -qec "${script_arg}" || {
211 die "Failed to recompile vpp-agent in Docker image!"
213 # Make sure .deb files of other version are not present.
214 rm_cmd="rm -vf /opt/vpp-agent/dev/vpp/build-root/vpp*.deb /opt/vpp/*.deb"
215 sudo docker exec agentcnt bash -c "${rm_cmd}" || {
216 die "Failed to remove VPP debian packages!"
218 for f in "${DOWNLOAD_DIR}"/*; do
219 sudo docker cp "$f" "agentcnt:/opt/vpp-agent/dev/vpp/build-root"/ || {
220 die "Failed to copy files to Docker image!"
223 # Save container state.
224 sudo docker commit "$(sudo docker ps -q)" "dev_vpp_agent:latest" || {
225 die "Failed to commit state of Docker image!"
228 # Build prod_vpp_agent docker image.
229 cd "docker/prod" || die "Change directory failed."
230 sudo docker build --tag "prod_vpp_agent" --no-cache "." || {
231 die "Failed to build Docker image!"
233 # Export Docker image.
234 sudo docker save "prod_vpp_agent" | gzip > "prod_vpp_agent.tar.gz" || {
235 die "Failed to save Docker image!"
237 docker_image="$(readlink -e "prod_vpp_agent.tar.gz")" || {
238 die "Failed to get Docker image path!"
240 rm -r "${DOWNLOAD_DIR}/vpp"* || die "Failed to remove VPP packages!"
241 mv "${docker_image}" "${DOWNLOAD_DIR}"/ || die "Failed to move image!"
245 function gather_vpp () {
248 # - BASH_FUNCTION_DIR - Bash directory with functions.
249 # - TEST_CODE - The test selection string from environment or argument.
250 # - DOWNLOAD_DIR - Path to directory pybot takes the build to test from.
251 # - CSIT_DIR - Path to existing root of local CSIT git repository.
253 # - VPP_VERSION - VPP stable version under test.
255 # - ${CSIT_DIR}/DPDK_STABLE_VER - DPDK version to use
256 # by csit-vpp not-timed jobs.
257 # - ${CSIT_DIR}/VPP_STABLE_VER_UBUNTU - Ubuntu VPP version to usee.
258 # - ../vpp*.deb|rpm - Relative to ${DOWNLOAD_DIR}, copied for vpp-csit jobs.
259 # Directories updated:
260 # - ${DOWNLOAD_DIR}, vpp-*.deb files are copied here for vpp-csit jobs.
261 # - ./ - Assumed ${DOWNLOAD_DIR}, vpp-*.deb|rpm files
262 # are downloaded here for csit-vpp.
264 # - die - Print to stderr and exit, defined in common_functions.sh
265 # Bash scripts executed:
266 # - ${CSIT_DIR}/resources/tools/scripts/download_install_vpp_pkgs.sh
267 # - Should download and extract requested files to ./.
271 case "${TEST_CODE}" in
272 # Not csit-vpp as this code is re-used by ligato gathering.
274 # Use downloaded packages with specific version.
275 if [[ "${TEST_CODE}" == *"daily"* ]] || \
276 ([[ "${TEST_CODE}" == *"weekly"* ]] && \
277 [[ "${TEST_CODE}" != *"device"* ]]) || \
278 [[ "${TEST_CODE}" == *"semiweekly"* ]];
280 warn "Downloading latest VPP packages from Packagecloud."
282 warn "Downloading stable VPP packages from Packagecloud."
283 VPP_VERSION="$(<"${CSIT_DIR}/${VPP_VER_FILE}")" || {
284 die "Read VPP stable version failed."
287 source "${BASH_FUNCTION_DIR}/artifacts.sh" || die "Source failed."
288 download_artifacts || die
291 # Use locally built packages.
292 mv "${DOWNLOAD_DIR}"/../"vpp"*".${PKG_SUFFIX}" "${DOWNLOAD_DIR}"/ || {
293 die "Move command failed."
297 die "Unable to identify job type from: ${TEST_CODE}"