1 # lib_apt.sh - Docker build script apt library.
4 # Copyright (c) 2021 Cisco and/or its affiliates.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at:
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 # Don't import more than once.
18 if [ -n "$(alias lib_apt_imported 2> /dev/null)" ] ; then
21 alias lib_apt_imported=true
23 export CIMAN_DOCKER_SCRIPTS=${CIMAN_DOCKER_SCRIPTS:-"$(dirname $BASH_SOURCE)"}
24 . "$CIMAN_DOCKER_SCRIPTS/lib_common.sh"
25 . "$CIMAN_DOCKER_SCRIPTS/lib_csit.sh"
27 dump_apt_package_list() {
28 branchname="$(echo $branch | sed -e 's,/,_,')"
30 "$DOCKER_BUILD_LOG_DIR/$FDIOTOOLS_IMAGENAME-$branchname-apt-packages.log"
33 apt_install_packages() {
34 apt-get install -y --allow-downgrades --allow-remove-essential \
35 --allow-change-held-packages $@
38 # Used for older OS distro's which are incompatible
39 # with modern distro cmake vesrion
40 apt_override_cmake_install_with_pip3_version() {
41 local os_cmake="/usr/bin/cmake"
42 local os_cmake_ver="$($os_cmake --version | head -1)"
43 local pip3_cmake="/usr/local/bin/cmake"
45 python3 -m pip --disable-pip-version-check install cmake || true
46 local pip3_cmake_ver="$($pip3_cmake --version | head -1)"
47 echo_log "Overriding $OS_NAME '$os_cmake_ver' with '$pip3_cmake_ver'!"
48 apt-get remove -y cmake --autoremove || true
49 update-alternatives --quiet --remove-all cmake || true
50 update-alternatives --quiet --install "$os_cmake" cmake "$pip3_cmake" 100
51 echo_log "Default cmake ($(which cmake)) version: '$(cmake --version | head -1)'!"
54 generate_apt_dockerfile_common() {
55 local executor_class="$1"
56 local executor_image="$2"
57 debian_docker_inst_sed="| sed -e 's/has_rootless_extras="1"//g' | sh
59 cat <<EOF >>"$DOCKERFILE"
61 # Create download dir to cache external tarballs
62 WORKDIR $DOCKER_DOWNLOADS_DIR
64 # Copy-in temporary build tree containing
65 # ci-management, vpp, & csit git repos
66 WORKDIR $DOCKER_BUILD_DIR
69 # Build Environment Variables
70 ENV DEBIAN_FRONTEND="noninteractive"
71 ENV FDIOTOOLS_IMAGE="$executor_image"
72 ENV FDIOTOOLS_EXECUTOR_CLASS="$executor_class"
73 ENV CIMAN_ROOT="$DOCKER_CIMAN_ROOT"
74 ENV PATH="\$PATH:$DOCKER_CIMAN_ROOT/docker/scripts"
77 RUN apt-get update -qq \\
78 && apt-get install -y \\
81 && sed -i 's/# \(en_US\.UTF-8 .*\)/\1/' /etc/locale.gen \\
82 && locale-gen en_US.UTF-8 \\
83 && dpkg-reconfigure --frontend=noninteractive locales \\
84 && update-locale LANG=en_US.UTF-8 \\
85 && TZ=Etc/UTC && ln -snf /usr/share/zoneinfo/\$TZ /etc/localtime && echo \$TZ > /etc/timezone \\
86 && rm -r /var/lib/apt/lists/*
87 ENV LANG="en_US.UTF-8" LANGUAGE="en_US" LC_ALL="en_US.UTF-8"
89 # Install baseline packages (minimum build & utils).
91 # ci-management global-jjb requirements:
99 # from packer/provision/baseline.sh:
111 # Python build from source requirements:
114 # TODO: Fix broken project requirement install targets
115 # graphviz for 'make bootstrap-doxygen' (VPP)
116 # doxygen for 'make doxygen' (VPP)
117 # enchant for 'make docs' (VPP)
118 # libffi-dev for python cffi install (Ubuntu20.04/VPP/aarch64)
119 # liblapack-dev for python numpy/scipy (CSIT/aarch64)
120 # libopenblas-dev for python numpy/scipy (CSIT/aarch64)
121 # libpcap-dev for python pypcap install (CSIT)
122 # sshpass for CSIT jobs
124 # From .../csit/resources/tools/presentation/run_report_*.sh:
132 # texlive-latex-recommended
133 # texlive-fonts-recommended
134 # texlive-fonts-extra
135 # texlive-latex-extra
140 RUN apt-get update -qq \\
141 && apt-get install -y \\
142 apt-transport-https \\
179 software-properties-common \\
182 texlive-fonts-extra \\
183 texlive-fonts-recommended \\
184 texlive-latex-extra \\
185 texlive-latex-recommended \\
196 && curl -L https://packagecloud.io/fdio/master/gpgkey | apt-key add - \\
197 && curl -s https://packagecloud.io/install/repositories/fdio/master/script.deb.sh | bash \\
199 # Hack to prevent failure on debian-9 build
201 if grep -qe 'debian:9' "$DOCKERFILE" ; then
202 echo " && curl -fsSL https://get.docker.com | sed -e 's/has_rootless_extras=\"1\"//g' | sh \ " >>"$DOCKERFILE"
204 echo " && curl -fsSL https://get.docker.com | sh \ " >>"$DOCKERFILE"
207 cat <<EOF >>"$DOCKERFILE"
208 && rm -r /var/lib/apt/lists/*
210 # Install packages for all project branches
212 RUN apt-get update -qq \\
213 && dbld_vpp_install_packages.sh \\
214 && dbld_csit_install_packages.sh \\
215 && rm -r /var/lib/apt/lists/*
219 generate_apt_dockerfile_clean() {
220 cat <<EOF >>"$DOCKERFILE"
222 # Clean up copy-in build tree
223 RUN dbld_dump_build_logs.sh \\
224 && rm -rf "/tmp/*" "$DOCKER_BUILD_FILES_DIR"
228 # Generate 'builder' class apt dockerfile
229 builder_generate_apt_dockerfile() {
230 local executor_class="$1"
231 local executor_os_name="$2"
232 local executor_image="$3"
233 local vpp_install_skip_sysctl_envvar="";
235 if grep -q "debian-9" <<< "$executor_os_name" ; then
236 # Workaround to VPP package installation failure on debian-9
237 vpp_install_skip_sysctl_envvar="ENV VPP_INSTALL_SKIP_SYSCTL=1"
239 generate_apt_dockerfile_common $executor_class $executor_image
240 csit_builder_generate_docker_build_files
241 cat <<EOF >>"$DOCKERFILE"
243 # Install LF-IT requirements
244 ENV LF_VENV="/root/lf-venv"
245 RUN apt-get update -qq \\
246 && dbld_lfit_requirements.sh \\
247 && rm -r /var/lib/apt/lists/*
249 # Install packagecloud requirements
250 RUN gem install rake package_cloud \\
251 && curl -s https://packagecloud.io/install/repositories/fdio/master/script.deb.sh | bash
253 # Install CSIT ssh requirements
254 # TODO: Verify why badkey is required & figure out how to avoid it.
255 COPY files/badkey /root/.ssh/id_rsa
256 COPY files/sshconfig /root/.ssh/config
258 # CI Runtime Environment
260 $vpp_install_skip_sysctl_envvar
261 ENV VPP_ZOMBIE_NOCHECK="1"
262 ENV CCACHE_DIR="/scratch/ccache"
264 generate_apt_dockerfile_clean
267 # Generate 'csit_dut' class apt dockerfile
268 csit_dut_generate_apt_dockerfile() {
269 local executor_class="$1"
270 local executor_os_name="$2"
271 local executor_image="$3"
273 csit_dut_generate_docker_build_files
274 generate_apt_dockerfile_common "$executor_class" "$executor_image"
275 cat <<EOF >>"$DOCKERFILE"
277 # Install csit_dut specific packages
278 RUN apt-get update -qq \\
279 && apt-get install -y \\
285 && rm -r /var/lib/apt/lists/*
288 RUN chown root:syslog /var/log \\
289 && chmod 755 /etc/default
291 # Create directory structure
292 RUN mkdir -p /var/run/sshd
295 RUN echo 'root:Csit1234' | chpasswd \\
296 && sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config \\
297 && sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
301 COPY files/supervisord.conf /etc/supervisor/supervisord.conf
303 CMD ["sh", "-c", "rm -f /dev/shm/db /dev/shm/global_vm /dev/shm/vpe-api; /usr/bin/supervisord -c /etc/supervisor/supervisord.conf; /usr/sbin/sshd -D -p 2222"]
305 generate_apt_dockerfile_clean
308 # Generate 'csit_shim' class apt dockerfile
309 csit_shim_generate_apt_dockerfile() {
310 local executor_class="$1"
311 local executor_os_name="$2"
312 local executor_image="$3"
314 csit_shim_generate_docker_build_files
315 cat <<EOF >>"$DOCKERFILE"
317 # Copy-in temporary build tree containing
318 # ci-management, vpp, & csit git repos
319 WORKDIR $DOCKER_BUILD_DIR
322 # Build Environment Variables
323 ENV DEBIAN_FRONTEND="noninteractive"
324 ENV FDIOTOOLS_IMAGE="$executor_image"
325 ENV FDIOTOOLS_EXECUTOR_CLASS="$executor_class"
326 ENV CIMAN_ROOT="$DOCKER_CIMAN_ROOT"
327 ENV PATH="\$PATH:$DOCKER_CIMAN_ROOT/docker/scripts"
329 # Configure locales & timezone
330 RUN apt-get update -qq \\
331 && apt-get install -y \\
334 && sed -i 's/# \(en_US\.UTF-8 .*\)/\1/' /etc/locale.gen \\
335 && locale-gen en_US.UTF-8 \\
336 && dpkg-reconfigure --frontend=noninteractive locales \\
337 && update-locale LANG=en_US.UTF-8 \\
338 && TZ=Etc/UTC && ln -snf /usr/share/zoneinfo/\$TZ /etc/localtime && echo \$TZ > /etc/timezone \\
339 && rm -r /var/lib/apt/lists/*
340 ENV LANG=en_US.UTF-8 LANGUAGE=en_US LC_ALL=en_US.UTF-8
342 COPY files/wrapdocker /usr/local/bin/wrapdocker
343 RUN chmod +x /usr/local/bin/wrapdocker
345 # Install packages and Docker
346 RUN apt-get update -qq \\
347 && apt-get install -y \\
356 && curl -fsSL https://get.docker.com | sh \\
357 && rm -rf /var/lib/apt/lists/*
359 RUN mkdir /var/run/sshd
360 RUN echo 'root:Csit1234' | chpasswd
361 RUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
363 # SSH login fix. Otherwise user is kicked off after login
364 RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
366 # Need volume for sidecar docker launches
367 VOLUME /var/lib/docker
369 # SSH to listen on port 6022 in shim
370 RUN echo 'Port 6022' >>/etc/ssh/sshd_config
371 RUN echo 'Port 6023' >>/etc/ssh/sshd_config
373 # TODO: Verify why badkeypub is required & figure out how to avoid it.
374 COPY files/badkeypub /root/.ssh/authorized_keys
375 COPY files/sshconfig /root/.ssh/config
377 # Clean up copy-in build tree
378 RUN rm -rf /tmp/* $DOCKER_BUILD_FILES_DIR
380 # Start sshd by default
382 CMD ["/usr/sbin/sshd", "-D"]
386 generate_apt_dockerfile() {
387 local executor_class="$1"
388 local executor_os_name="$2"
389 local from_image="$3"
390 local executor_image="$4"
392 cat <<EOF >"$DOCKERIGNOREFILE"
396 cat <<EOF >"$DOCKERFILE"
397 FROM $from_image AS ${executor_class}-executor-image
398 LABEL Description="FD.io CI '$executor_class' executor docker image for $executor_os_name/$OS_ARCH"
400 LABEL Version="$DOCKER_TAG"
402 ${executor_class}_generate_apt_dockerfile "$executor_class" \
403 "$executor_os_name" "$executor_image"