1 # lib_apt.sh - Docker build script apt library.
4 # Copyright (c) 2021 Cisco and/or its affiliates.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at:
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 # Don't import more than once.
18 if [ -n "$(alias lib_apt_imported 2> /dev/null)" ] ; then
21 alias lib_apt_imported=true
23 export CIMAN_DOCKER_SCRIPTS=${CIMAN_DOCKER_SCRIPTS:-"$(dirname $BASH_SOURCE)"}
24 . "$CIMAN_DOCKER_SCRIPTS/lib_common.sh"
25 . "$CIMAN_DOCKER_SCRIPTS/lib_csit.sh"
27 dump_apt_package_list() {
28 branchname="$(echo $branch | sed -e 's,/,_,')"
30 "$DOCKER_BUILD_LOG_DIR/$FDIOTOOLS_IMAGENAME-$branchname-apt-packages.log"
33 apt_install_packages() {
34 apt-get install -y --allow-downgrades --allow-remove-essential \
35 --allow-change-held-packages $@
38 # Used for older OS distro's which are incompatible
39 # with modern distro cmake vesrion
40 apt_override_cmake_install_with_pip3_version() {
41 local os_cmake="/usr/bin/cmake"
42 local os_cmake_ver="$($os_cmake --version | head -1)"
43 local pip3_cmake="/usr/local/bin/cmake"
45 python3 -m pip --disable-pip-version-check install cmake || true
46 local pip3_cmake_ver="$($pip3_cmake --version | head -1)"
47 echo_log "Overriding $OS_NAME '$os_cmake_ver' with '$pip3_cmake_ver'!"
48 apt-get remove -y cmake --autoremove || true
49 update-alternatives --quiet --remove-all cmake || true
50 update-alternatives --quiet --install "$os_cmake" cmake "$pip3_cmake" 100
51 echo_log "Default cmake ($(which cmake)) version: '$(cmake --version | head -1)'!"
54 generate_apt_dockerfile_common() {
55 local executor_class="$1"
56 local executor_image="$2"
57 local dpkg_arch="$(dpkg --print-architecture)"
59 cat <<EOF >>"$DOCKERFILE"
61 # Create download dir to cache external tarballs
62 WORKDIR $DOCKER_DOWNLOADS_DIR
64 # Copy-in temporary build tree containing
65 # ci-management, vpp, & csit git repos
66 WORKDIR $DOCKER_BUILD_DIR
69 # Build Environment Variables
70 ENV DEBIAN_FRONTEND="noninteractive"
71 ENV FDIOTOOLS_IMAGE="$executor_image"
72 ENV FDIOTOOLS_EXECUTOR_CLASS="$executor_class"
73 ENV CIMAN_ROOT="$DOCKER_CIMAN_ROOT"
74 ENV PATH="\$PATH:$DOCKER_CIMAN_ROOT/docker/scripts"
77 RUN apt-get update -qq \\
78 && apt-get install -y \\
81 && sed -i 's/# \(en_US\.UTF-8 .*\)/\1/' /etc/locale.gen \\
82 && locale-gen en_US.UTF-8 \\
83 && dpkg-reconfigure --frontend=noninteractive locales \\
84 && update-locale LANG=en_US.UTF-8 \\
85 && TZ=Etc/UTC && ln -snf /usr/share/zoneinfo/\$TZ /etc/localtime && echo \$TZ > /etc/timezone \\
86 && rm -r /var/lib/apt/lists/*
87 ENV LANG="en_US.UTF-8" LANGUAGE="en_US" LC_ALL="en_US.UTF-8"
89 # Install baseline packages (minimum build & utils).
91 # ci-management global-jjb requirements:
99 # from packer/provision/baseline.sh:
111 # Python build from source requirements:
114 # TODO: Fix broken project requirement install targets
115 # graphviz for 'make bootstrap-doxygen' (VPP)
116 # doxygen for 'make doxygen' (VPP)
117 # enchant for 'make docs' (VPP)
118 # libffi-dev for python cffi install (Ubuntu20.04/VPP/aarch64)
119 # liblapack-dev for python numpy/scipy (CSIT/aarch64)
120 # libopenblas-dev for python numpy/scipy (CSIT/aarch64)
121 # libpcap-dev for python pypcap install (CSIT)
122 # sshpass for CSIT jobs
124 # From .../csit/resources/tools/presentation/run_report_*.sh:
132 # texlive-latex-recommended
133 # texlive-fonts-recommended
134 # texlive-fonts-extra
135 # texlive-latex-extra
140 RUN apt-get update -qq \\
141 && apt-get install -y \\
142 apt-transport-https \\
179 software-properties-common \\
183 texlive-fonts-extra \\
184 texlive-fonts-recommended \\
185 texlive-latex-extra \\
186 texlive-latex-recommended \\
197 && curl -L https://packagecloud.io/fdio/master/gpgkey | apt-key add - \\
198 && curl -s https://packagecloud.io/install/repositories/fdio/master/script.deb.sh | bash \\
201 # Docker installation script fails on debian-9, so don't install docker
203 if ! grep -qe 'debian:9' "$DOCKERFILE" ; then
204 echo " && curl -fsSL https://get.docker.com | sh \ " >>"$DOCKERFILE"
207 cat <<EOF >>"$DOCKERFILE"
208 && rm -r /var/lib/apt/lists/*
210 # Install terraform for CSIT
212 RUN wget https://releases.hashicorp.com/terraform/1.0.4/terraform_1.0.4_linux_$dpkg_arch.zip \\
213 && unzip terraform_1.0.4_linux_$dpkg_arch.zip \\
214 && mv terraform /usr/bin \\
215 && rm -f terraform_1.0.4_linux_$dpkg_arch.zip
217 # Install packages for all project branches
219 RUN apt-get update -qq \\
220 && dbld_vpp_install_packages.sh \\
221 && dbld_csit_install_packages.sh \\
222 && rm -r /var/lib/apt/lists/*
226 generate_apt_dockerfile_clean() {
227 cat <<EOF >>"$DOCKERFILE"
229 # Clean up copy-in build tree
230 RUN dbld_dump_build_logs.sh \\
231 && rm -rf "/tmp/*" "$DOCKER_BUILD_FILES_DIR" "/root/.ccache"
235 # Generate 'builder' class apt dockerfile
236 builder_generate_apt_dockerfile() {
237 local executor_class="$1"
238 local executor_os_name="$2"
239 local executor_image="$3"
240 local vpp_install_skip_sysctl_envvar="";
242 if grep -q "debian-9" <<< "$executor_os_name" ; then
243 # Workaround to VPP package installation failure on debian-9
244 vpp_install_skip_sysctl_envvar="ENV VPP_INSTALL_SKIP_SYSCTL=1"
246 generate_apt_dockerfile_common $executor_class $executor_image
247 csit_builder_generate_docker_build_files
248 cat <<EOF >>"$DOCKERFILE"
250 # Install LF-IT requirements
251 ENV LF_VENV="/root/lf-venv"
252 RUN apt-get update -qq \\
253 && dbld_lfit_requirements.sh \\
254 && rm -r /var/lib/apt/lists/*
256 # Install packagecloud requirements
257 RUN gem install rake package_cloud \\
258 && curl -s https://packagecloud.io/install/repositories/fdio/master/script.deb.sh | bash
260 # Install CSIT ssh requirements
261 # TODO: Verify why badkey is required & figure out how to avoid it.
262 COPY files/badkey /root/.ssh/id_rsa
263 COPY files/sshconfig /root/.ssh/config
265 # CI Runtime Environment
267 $vpp_install_skip_sysctl_envvar
268 ENV VPP_ZOMBIE_NOCHECK="1"
269 ENV CCACHE_DIR="/scratch/ccache"
270 ENV CCACHE_MAXSIZE="10G"
272 generate_apt_dockerfile_clean
275 # Generate 'csit_dut' class apt dockerfile
276 csit_dut_generate_apt_dockerfile() {
277 local executor_class="$1"
278 local executor_os_name="$2"
279 local executor_image="$3"
281 csit_dut_generate_docker_build_files
282 generate_apt_dockerfile_common "$executor_class" "$executor_image"
283 cat <<EOF >>"$DOCKERFILE"
285 # Install csit_dut specific packages
286 RUN apt-get update -qq \\
287 && apt-get install -y \\
293 && rm -r /var/lib/apt/lists/*
296 RUN chown root:syslog /var/log \\
297 && chmod 755 /etc/default
299 # Create directory structure
300 RUN mkdir -p /var/run/sshd
303 RUN echo 'root:Csit1234' | chpasswd \\
304 && sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config \\
305 && sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
309 COPY files/supervisord.conf /etc/supervisor/supervisord.conf
311 CMD ["sh", "-c", "rm -f /dev/shm/db /dev/shm/global_vm /dev/shm/vpe-api; /usr/bin/supervisord -c /etc/supervisor/supervisord.conf; /usr/sbin/sshd -D -p 2222"]
313 generate_apt_dockerfile_clean
316 # Generate 'csit_shim' class apt dockerfile
317 csit_shim_generate_apt_dockerfile() {
318 local executor_class="$1"
319 local executor_os_name="$2"
320 local executor_image="$3"
322 csit_shim_generate_docker_build_files
323 cat <<EOF >>"$DOCKERFILE"
325 # Copy-in temporary build tree containing
326 # ci-management, vpp, & csit git repos
327 WORKDIR $DOCKER_BUILD_DIR
330 # Build Environment Variables
331 ENV DEBIAN_FRONTEND="noninteractive"
332 ENV FDIOTOOLS_IMAGE="$executor_image"
333 ENV FDIOTOOLS_EXECUTOR_CLASS="$executor_class"
334 ENV CIMAN_ROOT="$DOCKER_CIMAN_ROOT"
335 ENV PATH="\$PATH:$DOCKER_CIMAN_ROOT/docker/scripts"
337 # Configure locales & timezone
338 RUN apt-get update -qq \\
339 && apt-get install -y \\
342 && sed -i 's/# \(en_US\.UTF-8 .*\)/\1/' /etc/locale.gen \\
343 && locale-gen en_US.UTF-8 \\
344 && dpkg-reconfigure --frontend=noninteractive locales \\
345 && update-locale LANG=en_US.UTF-8 \\
346 && TZ=Etc/UTC && ln -snf /usr/share/zoneinfo/\$TZ /etc/localtime && echo \$TZ > /etc/timezone \\
347 && rm -r /var/lib/apt/lists/*
348 ENV LANG=en_US.UTF-8 LANGUAGE=en_US LC_ALL=en_US.UTF-8
350 COPY files/wrapdocker /usr/local/bin/wrapdocker
351 RUN chmod +x /usr/local/bin/wrapdocker
353 # Install packages and Docker
354 RUN apt-get update -qq \\
355 && apt-get install -y \\
364 && curl -fsSL https://get.docker.com | sh \\
365 && rm -rf /var/lib/apt/lists/*
367 RUN mkdir /var/run/sshd
368 RUN echo 'root:Csit1234' | chpasswd
369 RUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
371 # SSH login fix. Otherwise user is kicked off after login
372 RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
374 # Need volume for sidecar docker launches
375 VOLUME /var/lib/docker
377 # SSH to listen on port 6022 in shim
378 RUN echo 'Port 6022' >>/etc/ssh/sshd_config
379 RUN echo 'Port 6023' >>/etc/ssh/sshd_config
381 # TODO: Verify why badkeypub is required & figure out how to avoid it.
382 COPY files/badkeypub /root/.ssh/authorized_keys
383 COPY files/sshconfig /root/.ssh/config
385 # Clean up copy-in build tree
386 RUN rm -rf /tmp/* $DOCKER_BUILD_FILES_DIR
388 # Start sshd by default
390 CMD ["/usr/sbin/sshd", "-D"]
394 generate_apt_dockerfile() {
395 local executor_class="$1"
396 local executor_os_name="$2"
397 local from_image="$3"
398 local executor_image="$4"
400 cat <<EOF >"$DOCKERIGNOREFILE"
404 cat <<EOF >"$DOCKERFILE"
405 FROM $from_image AS ${executor_class}-executor-image
406 LABEL Description="FD.io CI '$executor_class' executor docker image for $executor_os_name/$OS_ARCH"
408 LABEL Version="$DOCKER_TAG"
410 ${executor_class}_generate_apt_dockerfile "$executor_class" \
411 "$executor_os_name" "$executor_image"