gerrit.fd Code Review - ci-management.git/blob

1 # lib_apt.sh - Docker build script apt library.

2 # For import only.

5 # Licensed under the Apache License, Version 2.0 (the "License");

6 # you may not use this file except in compliance with the License.

7 # You may obtain a copy of the License at:

8 #

9 # http://www.apache.org/licenses/LICENSE-2.0

10 #

11 # Unless required by applicable law or agreed to in writing, software

12 # distributed under the License is distributed on an "AS IS" BASIS,

13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

14 # See the License for the specific language governing permissions and

15 # limitations under the License.

17 # Don't import more than once.

18 if [ -n "$(alias lib_apt_imported 2> /dev/null)" ] ; then

19 return 0

20 fi

21 alias lib_apt_imported=true

23 DIND_FROM_IMAGE="cruizba/ubuntu-dind:jammy-26.1.3-r2"

24 HST_FROM_IMAGE="ubuntu:22.04"

26 export CIMAN_DOCKER_SCRIPTS=${CIMAN_DOCKER_SCRIPTS:-"$(dirname $BASH_SOURCE)"}

27 . "$CIMAN_DOCKER_SCRIPTS/lib_common.sh"

28 . "$CIMAN_DOCKER_SCRIPTS/lib_csit.sh"

30 dump_apt_package_list() {

31 branchname="$(echo $branch | sed -e 's,/,_,')"

32 dpkg -l > \

33 "$DOCKER_BUILD_LOG_DIR/$FDIOTOOLS_IMAGENAME-$branchname-apt-packages.log"

34 }

36 apt_install_packages() {

37 apt-get install -y --allow-downgrades --allow-remove-essential \

38 --allow-change-held-packages $@

39 }

41 generate_apt_dockerfile_common() {

42 local executor_class="$1"

43 local executor_image="$2"

44 local dpkg_arch="$(dpkg --print-architecture)"

46 cat <<EOF >>"$DOCKERFILE"

48 # Create download dir to cache external tarballs

49 WORKDIR $DOCKER_DOWNLOADS_DIR

51 # Copy-in temporary build tree containing

52 # ci-management, vpp, & csit git repos

53 WORKDIR $DOCKER_BUILD_DIR

54 COPY . .

56 # Build Environment Variables

57 ENV DEBIAN_FRONTEND="noninteractive"

58 ENV FDIOTOOLS_IMAGE="$executor_image"

59 ENV FDIOTOOLS_EXECUTOR_CLASS="$executor_class"

60 ENV CIMAN_ROOT="$DOCKER_CIMAN_ROOT"

61 ENV PATH="\$PATH:$DOCKER_CIMAN_ROOT/docker/scripts"

63 # Configure locales

64 RUN apt-get update -qq \\

65 && apt-get install -y \\

66 apt-utils \\

67 locales \\

68 && sed -i 's/# $en_US\.UTF-8 .*$/\1/' /etc/locale.gen \\

69 && locale-gen en_US.UTF-8 \\

70 && dpkg-reconfigure --frontend=noninteractive locales \\

71 && update-locale LANG=en_US.UTF-8 \\

72 && TZ=Etc/UTC && ln -snf /usr/share/zoneinfo/\$TZ /etc/localtime && echo \$TZ > /etc/timezone \\

73 && rm -r /var/lib/apt/lists/*

74 ENV LANG="en_US.UTF-8" LANGUAGE="en_US" LC_ALL="en_US.UTF-8"

76 # Install baseline packages (minimum build & utils).

77 #

78 # ci-management global-jjb requirements:

79 # facter

80 # python3-pip

81 # python3-venv

82 # for lftools:

83 # xmlstarlet

84 # libxml2-dev

85 # libxslt-dev

86 # from packer/provision/baseline.sh:

87 # unzip

88 # xz-utils

89 # git

90 # git-review

91 # libxml2-dev

92 # libxml-xpath-perl

93 # libxslt-dev

94 # make

95 # wget

96 # jq

97 #

98 # Python build from source requirements:

99 # build-essential

100 #

101 # TODO: Fix broken project requirement install targets

102 # graphviz for doxygen (HICN)

103 # doxygen for doxygen (HICN)

104 # libffi-dev for python cffi install (Ubuntu20.04/VPP/aarch64)

105 # liblapack-dev for python numpy/scipy (CSIT/aarch64)

106 # libopenblas-dev for python numpy/scipy (CSIT/aarch64)

107 # libpcap-dev for python pypcap install (CSIT)

108 # sshpass for CSIT jobs

109 #

110 RUN apt-get update -qq \\

111 && apt-get install -y \\

112 apt-transport-https \\

113 curl \\

114 ca-certificates \\

115 default-jdk \\

116 default-jre \\

117 dnsutils \\

118 doxygen \\

119 emacs \\

120 facter \\

121 gawk \\

122 gdb \\

123 gfortran \\

124 git \\

125 git-review \\

126 gnupg-agent \\

127 graphviz \\

128 iproute2 \\

129 iputils-clockdiff \\

130 iputils-ping \\

131 iputils-tracepath \\

132 jq \\

133 libffi-dev \\

134 liblapack-dev \\

135 libopenblas-dev \\

136 libpcap-dev \\

137 libxml-xpath-perl \\

138 make \\

139 python3-pip \\

140 python3-venv \\

141 python3-virtualenv \\

142 rsync \\

143 ruby-dev \\

144 software-properties-common \\

145 sshpass \\

146 sysstat \\

147 sudo \\

148 traceroute \\

149 tree \\

150 vim \\

151 wget \\

152 xmlstarlet \\

153 xz-utils \\

154 && curl -L https://packagecloud.io/fdio/master/gpgkey | apt-key add - \\

155 && curl -s https://packagecloud.io/install/repositories/fdio/master/script.deb.sh | bash \\

156 && rm -r /var/lib/apt/lists/*

157

158 # Install terraform for CSIT

159 #

160 RUN wget https://releases.hashicorp.com/terraform/1.7.3/terraform_1.7.3_linux_$dpkg_arch.zip \\

161 && unzip terraform_1.7.3_linux_$dpkg_arch.zip \\

162 && mv terraform /usr/bin \\

163 && rm -f terraform_1.7.3_linux_$dpkg_arch.zip

164 EOF

165

166 generate_apt_dockerfile_install_golang

167

168 cat <<EOF >>"$DOCKERFILE"

169

170 # Install packages for all project branches

171 #

172 RUN apt-get update -qq \\

173 && dbld_vpp_install_packages.sh \\

174 && dbld_csit_install_packages.sh \\

175 && apt-get install -y pkg-config \\

176 && rm -r /var/lib/apt/lists/*

177 EOF

178 }

179

180 generate_apt_dockerfile_clean() {

181 cat <<EOF >>"$DOCKERFILE"

182

183 # Clean up copy-in build tree

184 RUN dbld_dump_build_logs.sh \\

185 && apt-get -y autoremove \\

186 && rm -rf "/tmp/*" "$DOCKER_BUILD_FILES_DIR" "/root/.ccache"

187 EOF

188 }

189

190 generate_apt_dockerfile_install_golang() {

191 local go_tarball_arch="amd64"

192

193 if [ "$OS_ARCH" = "aarch64" ] ; then

194 go_tarball_arch="arm64"

195 fi

196

197 cat <<EOF >>"$DOCKERFILE"

198

199 # Install golang for HostStack Test (HST) jobs

200 #

201 ENV GOPATH /go

202 ENV GOROOT /usr/local/go

203 ENV PATH \$GOPATH/bin:/usr/local/go/bin:\$PATH

204 RUN rm -rf /usr/local/go /usr/bin/go \\

205 && wget -P /tmp "https://go.dev/dl/go${DOCKER_GOLANG_VERSION}.linux-${go_tarball_arch}.tar.gz" \\

206 && tar -C /usr/local -xzf "/tmp/go${DOCKER_GOLANG_VERSION}.linux-${go_tarball_arch}.tar.gz" \\

207 && rm "/tmp/go${DOCKER_GOLANG_VERSION}.linux-${go_tarball_arch}.tar.gz" \\

208 && ln -s /usr/local/go/bin/go /usr/bin/go \\

209 && echo -n "\nGOPATH=\$GOPATH\nGOROOT=\$GOROOT" | tee -a /etc/environment \\

210 && mkdir -p "\$GOPATH/src" "\$GOPATH/bin" && chmod -R 777 "\$GOPATH"

211 EOF

212 }

213

214 # Generate 'builder' class apt dockerfile

215 builder_generate_apt_dockerfile() {

216 local executor_class="$1"

217 local executor_os_name="$2"

218 local executor_image="$3"

219 local vpp_install_skip_sysctl_envvar="";

220

221 generate_apt_dockerfile_common "$executor_class" "$executor_image"

222 csit_builder_generate_docker_build_files

223 cat <<EOF >>"$DOCKERFILE"

224

225 # Install LF-IT requirements

226 ENV LF_VENV="/root/lf-venv"

227 RUN apt-get update -qq \\

228 && dbld_lfit_requirements.sh \\

229 && rm -r /var/lib/apt/lists/*

230

231 # Install packagecloud requirements

232 RUN gem install rake package_cloud \\

233 && curl -s https://packagecloud.io/install/repositories/fdio/master/script.deb.sh | bash

234

235 # Install CSIT ssh requirements

236 # TODO: Verify why badkey is required & figure out how to avoid it.

237 COPY files/badkey /root/.ssh/id_rsa

238 COPY files/sshconfig /root/.ssh/config

239

240 # CI Runtime Environment

241 WORKDIR /

242 $vpp_install_skip_sysctl_envvar

243 ENV VPP_ZOMBIE_NOCHECK="1"

244 ENV CCACHE_DIR="/scratch/ccache"

245 ENV CCACHE_MAXSIZE="10G"

246 EOF

247 generate_apt_dockerfile_clean

248 }

249

250 # Generate 'csit_dut' class apt dockerfile

251 csit_dut_generate_apt_dockerfile() {

252 local executor_class="$1"

253 local executor_os_name="$2"

254 local executor_image="$3"

255

256 csit_dut_generate_docker_build_files

257 generate_apt_dockerfile_common "$executor_class" "$executor_image"

258 cat <<EOF >>"$DOCKERFILE"

259

260 # Install csit_dut specific packages

261 RUN apt-get update -qq \\

262 && apt-get install -y \\

263 net-tools \\

264 openssh-server \\

265 pciutils \\

266 rsyslog \\

267 supervisor \\

268 && rm -r /var/lib/apt/lists/*

269

270 # Fix permissions

271 RUN chown root:syslog /var/log \\

272 && chmod 755 /etc/default

273

274 # Create directory structure

275 RUN mkdir -p /var/run/sshd

276

277 # SSH settings

278 RUN echo 'root:Csit1234' | chpasswd \\

279 && sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config \\

280 && sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd

281

282 EXPOSE 2222

283

284 COPY files/supervisord.conf /etc/supervisor/supervisord.conf

285

286 CMD ["sh", "-c", "rm -f /dev/shm/db /dev/shm/global_vm /dev/shm/vpe-api; /usr/bin/supervisord -c /etc/supervisor/supervisord.conf; /usr/sbin/sshd -D -p 2222"]

287 EOF

288 generate_apt_dockerfile_clean

289 }

290

291 # Generate 'csit_shim' class apt dockerfile

292 csit_shim_generate_apt_dockerfile() {

293 local executor_class="$1"

294 local executor_os_name="$2"

295 local executor_image="$3"

296

297 csit_shim_generate_docker_build_files

298 cat <<EOF >>"$DOCKERFILE"

299

300 # Copy-in temporary build tree containing

301 # ci-management, vpp, & csit git repos

302 WORKDIR $DOCKER_BUILD_DIR

303 COPY . .

304

305 # Build Environment Variables

306 ENV DEBIAN_FRONTEND="noninteractive"

307 ENV FDIOTOOLS_IMAGE="$executor_image"

308 ENV FDIOTOOLS_EXECUTOR_CLASS="$executor_class"

309 ENV CIMAN_ROOT="$DOCKER_CIMAN_ROOT"

310 ENV PATH="\$PATH:$DOCKER_CIMAN_ROOT/docker/scripts"

311

312 # Configure locales & timezone

313 RUN apt-get update -qq \\

314 && apt-get install -y \\

315 apt-utils \\

316 locales \\

317 && sed -i 's/# $en_US\.UTF-8 .*$/\1/' /etc/locale.gen \\

318 && locale-gen en_US.UTF-8 \\

319 && dpkg-reconfigure --frontend=noninteractive locales \\

320 && update-locale LANG=en_US.UTF-8 \\

321 && TZ=Etc/UTC && ln -snf /usr/share/zoneinfo/\$TZ /etc/localtime && echo \$TZ > /etc/timezone \\

322 && rm -r /var/lib/apt/lists/*

323 ENV LANG=en_US.UTF-8 LANGUAGE=en_US LC_ALL=en_US.UTF-8

324

325 COPY files/wrapdocker /usr/local/bin/wrapdocker

326 RUN chmod +x /usr/local/bin/wrapdocker

327

328 # Install packages and Docker

329 RUN apt-get update -qq \\

330 && apt-get install -y \\

331 bash \\

332 curl \\

333 iproute2 \\

334 locales \\

335 ssh \\

336 sudo \\

337 tzdata \\

338 uuid-runtime \\

339 && curl -fsSL https://get.docker.com | sh \\

340 && rm -rf /var/lib/apt/lists/*

341

342 RUN mkdir /var/run/sshd

343 RUN echo 'root:Csit1234' | chpasswd

344 RUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config

345

346 # SSH login fix. Otherwise user is kicked off after login

347 RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd

348

349 # Need volume for sidecar docker launches

350 VOLUME /var/lib/docker

351

352 # SSH to listen on port 6022 in shim

353 RUN echo 'Port 6022' >>/etc/ssh/sshd_config

354 RUN echo 'Port 6023' >>/etc/ssh/sshd_config

355

356 # TODO: Verify why badkeypub is required & figure out how to avoid it.

357 COPY files/badkeypub /root/.ssh/authorized_keys

358 COPY files/sshconfig /root/.ssh/config

359

360 # Clean up copy-in build tree

361 RUN rm -rf /tmp/* $DOCKER_BUILD_FILES_DIR

362

363 # Start sshd by default

364 EXPOSE 22

365 CMD ["/usr/sbin/sshd", "-D"]

366 EOF

367 }

368

369 generate_apt_dockerfile() {

370 local executor_class="$1"

371 local executor_os_name="$2"

372 local from_image="$3"

373 local executor_image="$4"

374 local is_dind_image="false"

375

376 # TODO: Enable HST on AARCH64 when supported in vpp/extras/hs-test

377 if [ "$from_image" = "$HST_FROM_IMAGE" ] ; then

378 from_image="$DIND_FROM_IMAGE"

379 is_dind_image="true"

380 fi

381

382 cat <<EOF >"$DOCKERIGNOREFILE"

383 **/__pycache__

384 *.pyc

385 EOF

386 cat <<EOF >"$DOCKERFILE"

387 FROM $from_image AS ${executor_class}-executor-image

388 LABEL Description="FD.io CI '$executor_class' executor docker image for $executor_os_name/$OS_ARCH"

389 LABEL Vendor="fd.io"

390 LABEL Version="$DOCKER_TAG"

391 EOF

392 if [ "$is_dind_image" = "true" ] ; then

393 cat <<EOF >>"$DOCKERFILE"

394 # Remove extraneous environment variable(s) created in "$from_image"

395 ENV DEBUG=

396 EOF

397 fi

398

399 "${executor_class}"_generate_apt_dockerfile "$executor_class" \

400 "$executor_os_name" "$executor_image"

401 }