1 # lib_apt.sh - Docker build script apt library.
4 # Copyright (c) 2023 Cisco and/or its affiliates.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at:
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 # Don't import more than once.
18 if [ -n "$(alias lib_apt_imported 2> /dev/null)" ] ; then
21 alias lib_apt_imported=true
23 export CIMAN_DOCKER_SCRIPTS=${CIMAN_DOCKER_SCRIPTS:-"$(dirname $BASH_SOURCE)"}
24 . "$CIMAN_DOCKER_SCRIPTS/lib_common.sh"
25 . "$CIMAN_DOCKER_SCRIPTS/lib_csit.sh"
27 dump_apt_package_list() {
28 branchname="$(echo $branch | sed -e 's,/,_,')"
30 "$DOCKER_BUILD_LOG_DIR/$FDIOTOOLS_IMAGENAME-$branchname-apt-packages.log"
33 apt_install_packages() {
34 apt-get install -y --allow-downgrades --allow-remove-essential \
35 --allow-change-held-packages $@
38 generate_apt_dockerfile_common() {
39 local executor_class="$1"
40 local executor_image="$2"
41 local dpkg_arch="$(dpkg --print-architecture)"
43 cat <<EOF >>"$DOCKERFILE"
45 # Create download dir to cache external tarballs
46 WORKDIR $DOCKER_DOWNLOADS_DIR
48 # Copy-in temporary build tree containing
49 # ci-management, vpp, & csit git repos
50 WORKDIR $DOCKER_BUILD_DIR
53 # Build Environment Variables
54 ENV DEBIAN_FRONTEND="noninteractive"
55 ENV FDIOTOOLS_IMAGE="$executor_image"
56 ENV FDIOTOOLS_EXECUTOR_CLASS="$executor_class"
57 ENV CIMAN_ROOT="$DOCKER_CIMAN_ROOT"
58 ENV PATH="\$PATH:$DOCKER_CIMAN_ROOT/docker/scripts"
61 RUN apt-get update -qq \\
62 && apt-get install -y \\
65 && sed -i 's/# \(en_US\.UTF-8 .*\)/\1/' /etc/locale.gen \\
66 && locale-gen en_US.UTF-8 \\
67 && dpkg-reconfigure --frontend=noninteractive locales \\
68 && update-locale LANG=en_US.UTF-8 \\
69 && TZ=Etc/UTC && ln -snf /usr/share/zoneinfo/\$TZ /etc/localtime && echo \$TZ > /etc/timezone \\
70 && rm -r /var/lib/apt/lists/*
71 ENV LANG="en_US.UTF-8" LANGUAGE="en_US" LC_ALL="en_US.UTF-8"
73 # Install baseline packages (minimum build & utils).
75 # ci-management global-jjb requirements:
83 # from packer/provision/baseline.sh:
95 # Python build from source requirements:
98 # TODO: Fix broken project requirement install targets
99 # graphviz for doxygen (HICN)
100 # doxygen for doxygen (HICN)
101 # libffi-dev for python cffi install (Ubuntu20.04/VPP/aarch64)
102 # liblapack-dev for python numpy/scipy (CSIT/aarch64)
103 # libopenblas-dev for python numpy/scipy (CSIT/aarch64)
104 # libpcap-dev for python pypcap install (CSIT)
105 # sshpass for CSIT jobs
107 # From .../csit/resources/tools/presentation/run_report_*.sh: // TO BE REMOVED IN RLS2306
115 # texlive-latex-recommended
116 # texlive-fonts-recommended
117 # texlive-fonts-extra
118 # texlive-latex-extra
123 RUN apt-get update -qq \\
124 && apt-get install -y \\
125 apt-transport-https \\
161 software-properties-common \\
165 texlive-fonts-extra \\
166 texlive-fonts-recommended \\
167 texlive-latex-extra \\
168 texlive-latex-recommended \\
179 && curl -L https://packagecloud.io/fdio/master/gpgkey | apt-key add - \\
180 && curl -s https://packagecloud.io/install/repositories/fdio/master/script.deb.sh | bash \\
183 cat <<EOF >>"$DOCKERFILE"
184 && rm -r /var/lib/apt/lists/*
186 # Install terraform for CSIT
188 RUN wget https://releases.hashicorp.com/terraform/1.4.2/terraform_1.4.2_linux_$dpkg_arch.zip \\
189 && unzip terraform_1.4.2_linux_$dpkg_arch.zip \\
190 && mv terraform /usr/bin \\
191 && rm -f terraform_1.4.2_linux_$dpkg_arch.zip
193 # Install Hugo and Go for CSIT
194 RUN wget https://github.com/gohugoio/hugo/releases/download/v0.111.3/hugo_extended_0.111.3_linux-$dpkg_arch.deb \\
195 && dpkg -i hugo_extended_0.111.3_linux-$dpkg_arch.deb \\
196 && rm -f hugo_extended_0.111.3_linux-$dpkg_arch.deb \\
197 && wget https://go.dev/dl/go1.20.2.linux-$dpkg_arch.tar.gz \\
198 && rm -rf /usr/local/go \\
199 && tar -C /usr/local -xzf go1.20.2.linux-$dpkg_arch.tar.gz \\
200 && rm -f go1.20.2.linux-$dpkg_arch.tar.gz
202 # Install packages for all project branches
204 RUN apt-get update -qq \\
205 && dbld_vpp_install_packages.sh \\
206 && dbld_csit_install_packages.sh \\
207 && rm -r /var/lib/apt/lists/*
211 generate_apt_dockerfile_clean() {
212 cat <<EOF >>"$DOCKERFILE"
214 # Clean up copy-in build tree
215 RUN dbld_dump_build_logs.sh \\
216 && rm -rf "/tmp/*" "$DOCKER_BUILD_FILES_DIR" "/root/.ccache"
220 # Generate 'builder' class apt dockerfile
221 builder_generate_apt_dockerfile() {
222 local executor_class="$1"
223 local executor_os_name="$2"
224 local executor_image="$3"
225 local vpp_install_skip_sysctl_envvar="";
227 generate_apt_dockerfile_common $executor_class $executor_image
228 csit_builder_generate_docker_build_files
229 cat <<EOF >>"$DOCKERFILE"
231 # Install LF-IT requirements
232 ENV LF_VENV="/root/lf-venv"
233 RUN apt-get update -qq \\
234 && dbld_lfit_requirements.sh \\
235 && rm -r /var/lib/apt/lists/*
237 # Install packagecloud requirements
238 RUN gem install rake package_cloud \\
239 && curl -s https://packagecloud.io/install/repositories/fdio/master/script.deb.sh | bash
241 # Install CSIT ssh requirements
242 # TODO: Verify why badkey is required & figure out how to avoid it.
243 COPY files/badkey /root/.ssh/id_rsa
244 COPY files/sshconfig /root/.ssh/config
246 # CI Runtime Environment
248 $vpp_install_skip_sysctl_envvar
249 ENV VPP_ZOMBIE_NOCHECK="1"
250 ENV CCACHE_DIR="/scratch/ccache"
251 ENV CCACHE_MAXSIZE="10G"
253 generate_apt_dockerfile_clean
256 # Generate 'csit_dut' class apt dockerfile
257 csit_dut_generate_apt_dockerfile() {
258 local executor_class="$1"
259 local executor_os_name="$2"
260 local executor_image="$3"
262 csit_dut_generate_docker_build_files
263 generate_apt_dockerfile_common "$executor_class" "$executor_image"
264 cat <<EOF >>"$DOCKERFILE"
266 # Install csit_dut specific packages
267 RUN apt-get update -qq \\
268 && apt-get install -y \\
274 && rm -r /var/lib/apt/lists/*
277 RUN chown root:syslog /var/log \\
278 && chmod 755 /etc/default
280 # Create directory structure
281 RUN mkdir -p /var/run/sshd
284 RUN echo 'root:Csit1234' | chpasswd \\
285 && sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config \\
286 && sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
290 COPY files/supervisord.conf /etc/supervisor/supervisord.conf
292 CMD ["sh", "-c", "rm -f /dev/shm/db /dev/shm/global_vm /dev/shm/vpe-api; /usr/bin/supervisord -c /etc/supervisor/supervisord.conf; /usr/sbin/sshd -D -p 2222"]
294 generate_apt_dockerfile_clean
297 # Generate 'csit_shim' class apt dockerfile
298 csit_shim_generate_apt_dockerfile() {
299 local executor_class="$1"
300 local executor_os_name="$2"
301 local executor_image="$3"
303 csit_shim_generate_docker_build_files
304 cat <<EOF >>"$DOCKERFILE"
306 # Copy-in temporary build tree containing
307 # ci-management, vpp, & csit git repos
308 WORKDIR $DOCKER_BUILD_DIR
311 # Build Environment Variables
312 ENV DEBIAN_FRONTEND="noninteractive"
313 ENV FDIOTOOLS_IMAGE="$executor_image"
314 ENV FDIOTOOLS_EXECUTOR_CLASS="$executor_class"
315 ENV CIMAN_ROOT="$DOCKER_CIMAN_ROOT"
316 ENV PATH="\$PATH:$DOCKER_CIMAN_ROOT/docker/scripts"
318 # Configure locales & timezone
319 RUN apt-get update -qq \\
320 && apt-get install -y \\
323 && sed -i 's/# \(en_US\.UTF-8 .*\)/\1/' /etc/locale.gen \\
324 && locale-gen en_US.UTF-8 \\
325 && dpkg-reconfigure --frontend=noninteractive locales \\
326 && update-locale LANG=en_US.UTF-8 \\
327 && TZ=Etc/UTC && ln -snf /usr/share/zoneinfo/\$TZ /etc/localtime && echo \$TZ > /etc/timezone \\
328 && rm -r /var/lib/apt/lists/*
329 ENV LANG=en_US.UTF-8 LANGUAGE=en_US LC_ALL=en_US.UTF-8
331 COPY files/wrapdocker /usr/local/bin/wrapdocker
332 RUN chmod +x /usr/local/bin/wrapdocker
334 # Install packages and Docker
335 RUN apt-get update -qq \\
336 && apt-get install -y \\
345 && curl -fsSL https://get.docker.com | sh \\
346 && rm -rf /var/lib/apt/lists/*
348 RUN mkdir /var/run/sshd
349 RUN echo 'root:Csit1234' | chpasswd
350 RUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
352 # SSH login fix. Otherwise user is kicked off after login
353 RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
355 # Need volume for sidecar docker launches
356 VOLUME /var/lib/docker
358 # SSH to listen on port 6022 in shim
359 RUN echo 'Port 6022' >>/etc/ssh/sshd_config
360 RUN echo 'Port 6023' >>/etc/ssh/sshd_config
362 # TODO: Verify why badkeypub is required & figure out how to avoid it.
363 COPY files/badkeypub /root/.ssh/authorized_keys
364 COPY files/sshconfig /root/.ssh/config
366 # Clean up copy-in build tree
367 RUN rm -rf /tmp/* $DOCKER_BUILD_FILES_DIR
369 # Start sshd by default
371 CMD ["/usr/sbin/sshd", "-D"]
375 generate_apt_dockerfile() {
376 local executor_class="$1"
377 local executor_os_name="$2"
378 local from_image="$3"
379 local executor_image="$4"
381 cat <<EOF >"$DOCKERIGNOREFILE"
385 cat <<EOF >"$DOCKERFILE"
386 FROM $from_image AS ${executor_class}-executor-image
387 LABEL Description="FD.io CI '$executor_class' executor docker image for $executor_os_name/$OS_ARCH"
389 LABEL Version="$DOCKER_TAG"
391 ${executor_class}_generate_apt_dockerfile "$executor_class" \
392 "$executor_os_name" "$executor_image"