From: Florin Coras Date: Mon, 2 Jun 2025 17:58:46 +0000 (-0400) Subject: hs-test: use base image container for container builds X-Git-Url: https://gerrit.fd.io/r/gitweb?a=commitdiff_plain;h=d63737c1b53c1157fed4a7278ea39d253b7ca4a8;p=vpp.git hs-test: use base image container for container builds Avoid multiple updates/downloads of dependencies. Type: improvement Change-Id: Ic770dc00a42cb5aa8c780aebb72beef390200363 Signed-off-by: Florin Coras Signed-off-by: Matus Fabian --- diff --git a/extras/hs-test/docker/Dockerfile.ab b/extras/hs-test/docker/Dockerfile.ab index 3ed1528c8a4..5e975af3b95 100644 --- a/extras/hs-test/docker/Dockerfile.ab +++ b/extras/hs-test/docker/Dockerfile.ab @@ -1,9 +1,7 @@ +# Apache Bench container that uses the base image ARG UBUNTU_VERSION=22.04 +FROM localhost:5000/vpp-test-base:latest -FROM ubuntu:${UBUNTU_VERSION} - -RUN apt-get update \ - && apt-get install -y apache2-utils \ - && rm -rf /var/lib/apt/lists/* +# apache2-utils is now installed in the base image ENTRYPOINT ["ab"] diff --git a/extras/hs-test/docker/Dockerfile.base b/extras/hs-test/docker/Dockerfile.base new file mode 100644 index 00000000000..12b7c0588a8 --- /dev/null +++ b/extras/hs-test/docker/Dockerfile.base @@ -0,0 +1,65 @@ +ARG UBUNTU_VERSION=22.04 +FROM ubuntu:${UBUNTU_VERSION} AS base + +# Set environment variables +ENV DEBIAN_FRONTEND=noninteractive + +# Install common dependencies needed across multiple containers +RUN apt-get update && apt-get install -y --no-install-recommends \ + # Basic utilities + ca-certificates \ + wget \ + gnupg \ + gnupg2 \ + git \ + iproute2 \ + iputils-ping \ + less \ + lsb-release \ + ubuntu-keyring \ + vim \ + # Development & debugging + gdb \ + libunwind-dev \ + # Libraries frequently needed + libapr1 \ + libnl-3-dev \ + libnl-route-3-dev \ + libnuma1 \ + libsubunit0 \ + openssl \ + python3 \ + # Tools used in tests + iperf3 \ + redis \ + redis-tools \ + xz-utils \ + # Tools moved from derived images + apache2-utils \ + nghttp2 \ + wrk + +# Because of http/3 we can't use stock curl in ubuntu 24.04 +ARG TARGETARCH +COPY script/build_curl.sh /build_curl.sh +RUN /build_curl.sh + +# Because of http/3 support we can't use stock nginx in ubuntu 24.04 +RUN curl https://nginx.org/keys/nginx_signing.key | gpg --dearmor \ +| tee /usr/share/keyrings/nginx-archive-keyring.gpg >/dev/null +RUN echo "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] \ + http://nginx.org/packages/ubuntu `lsb_release -cs` nginx" \ + | tee /etc/apt/sources.list.d/nginx.list +RUN bash -c 'echo -e "Package: *\nPin: origin nginx.org\nPin: release o=nginx\nPin-Priority: 900\n" \ +| tee /etc/apt/preferences.d/99nginx' + +RUN apt update && apt install -y nginx=1.26.2* \ + # Clean up + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Add a non-root user (useful for containers that need it) +RUN groupadd -r vpp && useradd -r -g vpp -s /bin/bash vpp + +# Set default command +CMD ["/bin/bash"] diff --git a/extras/hs-test/docker/Dockerfile.curl b/extras/hs-test/docker/Dockerfile.curl index 8e9b579aad4..b76a9d8da66 100644 --- a/extras/hs-test/docker/Dockerfile.curl +++ b/extras/hs-test/docker/Dockerfile.curl @@ -1,16 +1,10 @@ +# curl container that uses the base image ARG UBUNTU_VERSION=22.04 +FROM localhost:5000/vpp-test-base:latest -FROM ubuntu:${UBUNTU_VERSION} +# Note: wget and xz-utils are already in the base image -ARG TARGETARCH - -RUN apt-get update \ - && apt-get install -y xz-utils wget \ - && rm -rf /var/lib/apt/lists/* - -COPY script/build_curl.sh /build_curl.sh COPY resources/curl/* /tmp/ RUN fallocate -l 10MB /tmp/testFile -RUN /build_curl.sh CMD ["/bin/sh"] diff --git a/extras/hs-test/docker/Dockerfile.h2load b/extras/hs-test/docker/Dockerfile.h2load index de9d083c007..40bfc72aae4 100644 --- a/extras/hs-test/docker/Dockerfile.h2load +++ b/extras/hs-test/docker/Dockerfile.h2load @@ -1,9 +1,7 @@ +# h2load container that uses the base image ARG UBUNTU_VERSION=22.04 +FROM localhost:5000/vpp-test-base:latest -FROM ubuntu:${UBUNTU_VERSION} - -RUN apt-get update \ - && apt-get install -y nghttp2 \ - && rm -rf /var/lib/apt/lists/* +# nghttp2 is now installed in the base image ENTRYPOINT ["h2load"] diff --git a/extras/hs-test/docker/Dockerfile.nginx b/extras/hs-test/docker/Dockerfile.nginx index fc85f00aaae..bc392d5f0c4 100644 --- a/extras/hs-test/docker/Dockerfile.nginx +++ b/extras/hs-test/docker/Dockerfile.nginx @@ -1,10 +1,8 @@ +# nginx container that uses the base image ARG UBUNTU_VERSION=22.04 +FROM localhost:5000/vpp-test-base:latest -FROM ubuntu:${UBUNTU_VERSION} - -RUN apt-get update \ - && apt-get install -y nginx gdb less libunwind-dev \ - && rm -rf /var/lib/apt/lists/* +# nginx is now installed in the base image COPY resources/nginx/nginx.conf /nginx.conf COPY script/nginx_ldp.sh /usr/bin/nginx_ldp.sh diff --git a/extras/hs-test/docker/Dockerfile.nginx-http3 b/extras/hs-test/docker/Dockerfile.nginx-http3 index bde73e32da1..568bd9baab1 100644 --- a/extras/hs-test/docker/Dockerfile.nginx-http3 +++ b/extras/hs-test/docker/Dockerfile.nginx-http3 @@ -1,18 +1,6 @@ ARG UBUNTU_VERSION=22.04 -FROM ubuntu:${UBUNTU_VERSION} - -RUN apt-get update \ - && apt-get install -y curl gnupg2 ca-certificates lsb-release ubuntu-keyring libunwind-dev -RUN curl https://nginx.org/keys/nginx_signing.key | gpg --dearmor \ -| tee /usr/share/keyrings/nginx-archive-keyring.gpg >/dev/null -RUN echo "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] \ - http://nginx.org/packages/ubuntu `lsb_release -cs` nginx" \ - | tee /etc/apt/sources.list.d/nginx.list -RUN bash -c 'echo -e "Package: *\nPin: origin nginx.org\nPin: release o=nginx\nPin-Priority: 900\n" \ -| tee /etc/apt/preferences.d/99nginx' - -RUN apt update && apt install -y nginx=1.26.2* +FROM localhost:5000/vpp-test-base:latest COPY resources/nginx/vcl.conf /vcl.conf COPY resources/nginx/nginx_http3.conf /nginx.conf diff --git a/extras/hs-test/docker/Dockerfile.nginx-server b/extras/hs-test/docker/Dockerfile.nginx-server index b245b41e1be..33c004a48bc 100644 --- a/extras/hs-test/docker/Dockerfile.nginx-server +++ b/extras/hs-test/docker/Dockerfile.nginx-server @@ -1,10 +1,8 @@ +# nginx server container that uses the base image ARG UBUNTU_VERSION=22.04 +FROM localhost:5000/vpp-test-base:latest -FROM ubuntu:${UBUNTU_VERSION} - -RUN apt-get update \ - && apt-get install -y nginx \ - && rm -rf /var/lib/apt/lists/* +# nginx is now installed in the base image COPY resources/nginx/nginx_server_mirroring.conf /nginx.conf COPY script/nginx_server_entrypoint.sh /usr/bin/nginx_server_entrypoint.sh diff --git a/extras/hs-test/docker/Dockerfile.vpp b/extras/hs-test/docker/Dockerfile.vpp index 0f897c8a0ff..69414c80d15 100644 --- a/extras/hs-test/docker/Dockerfile.vpp +++ b/extras/hs-test/docker/Dockerfile.vpp @@ -1,12 +1,9 @@ +# VPP container that uses the base image ARG UBUNTU_VERSION=22.04 +FROM localhost:5000/vpp-test-base:latest -FROM ubuntu:${UBUNTU_VERSION} - -RUN apt-get update \ - && apt-get install -y openssl libapr1 libnuma1 libsubunit0 \ - iproute2 libnl-3-dev libnl-route-3-dev python3 iputils-ping \ - vim gdb libunwind-dev redis redis-tools iperf3 \ - && rm -rf /var/lib/apt/lists/* +# We don't need to install these packages as they're in the base image +# Just install anything specific to VPP that isn't in the base ARG OS_ARCH RUN echo "I'm building for $OS_ARCH" @@ -34,6 +31,6 @@ COPY vpp-data/bin/vpp_echo /usr/bin/ COPY vpp-data/bin/vcl_* /usr/bin/ COPY vpp-data/lib/*.so /usr/lib/ -RUN addgroup vpp +# Group already created in base image ENTRYPOINT ["tail", "-f", "/dev/null"] diff --git a/extras/hs-test/docker/Dockerfile.wrk b/extras/hs-test/docker/Dockerfile.wrk index b4108732556..ae376c5af8d 100644 --- a/extras/hs-test/docker/Dockerfile.wrk +++ b/extras/hs-test/docker/Dockerfile.wrk @@ -1,9 +1,7 @@ +# wrk HTTP benchmarking container that uses the base image ARG UBUNTU_VERSION=22.04 +FROM localhost:5000/vpp-test-base:latest -FROM ubuntu:${UBUNTU_VERSION} - -RUN apt-get update \ - && apt-get install -y wrk \ - && rm -rf /var/lib/apt/lists/* +# wrk is installed in the base image ENTRYPOINT ["wrk"] diff --git a/extras/hs-test/docker/setup-local-registry.sh b/extras/hs-test/docker/setup-local-registry.sh new file mode 100755 index 00000000000..684f858584a --- /dev/null +++ b/extras/hs-test/docker/setup-local-registry.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# Script to set up a local Docker registry + +set -e + +# Check if Docker is running +if ! docker info &>/dev/null; then + echo "Error: Docker is not running. Please start Docker and try again." + exit 1 +fi + +# Registry container name +REGISTRY_NAME="local-registry" +REGISTRY_PORT=5000 + +# Check if registry container is already running +if docker container inspect "$REGISTRY_NAME" &>/dev/null; then + echo "=== Local registry '$REGISTRY_NAME' is already running ===" + REGISTRY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$REGISTRY_NAME") + echo "Registry is available at: localhost:$REGISTRY_PORT or $REGISTRY_IP:$REGISTRY_PORT" +else + echo "=== Setting up local Docker registry ===" + + # Create a new registry container + docker run -d \ + --name "$REGISTRY_NAME" \ + --restart=always \ + -p "$REGISTRY_PORT:5000" \ + -v /var/lib/registry:/var/lib/registry \ + registry:2 + + REGISTRY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$REGISTRY_NAME") + echo "Registry container created successfully!" + echo "Registry is available at: localhost:$REGISTRY_PORT or $REGISTRY_IP:$REGISTRY_PORT" + + # Configure Docker to trust this insecure registry + echo "=== Configuring Docker to trust insecure registry ===" + if [ -f /etc/docker/daemon.json ]; then + # Check if the file already has an insecure-registries entry + if grep -q "insecure-registries" /etc/docker/daemon.json; then + echo "Insecure registries already configured. Please make sure 'localhost:$REGISTRY_PORT' is included." + else + echo "Adding 'localhost:$REGISTRY_PORT' to insecure-registries in /etc/docker/daemon.json" + echo "You may need to restart Docker for changes to take effect" + echo "Please add the following to /etc/docker/daemon.json:" + echo '{ + "insecure-registries": ["localhost:5000"] +}' + fi + else + echo "Creating /etc/docker/daemon.json with insecure-registries configuration" + echo "You may need to restart Docker for changes to take effect" + echo "Please create /etc/docker/daemon.json with the following content:" + echo '{ + "insecure-registries": ["localhost:5000"] +}' + fi +fi + +echo "" +echo "=== Local Registry Setup Complete ===" +echo "To use the local registry, prefix your image tags with 'localhost:$REGISTRY_PORT/'" +echo "For example: localhost:$REGISTRY_PORT/hs-test/vpp:latest" diff --git a/extras/hs-test/infra/hst_suite.go b/extras/hs-test/infra/hst_suite.go index 2fb05a48d8c..6190e444d8e 100644 --- a/extras/hs-test/infra/hst_suite.go +++ b/extras/hs-test/infra/hst_suite.go @@ -83,6 +83,7 @@ var reservedPorts = []string{ "4789", "48879", "4790", + "5000", "6633", "6081", "53053", diff --git a/extras/hs-test/script/build-images.sh b/extras/hs-test/script/build-images.sh new file mode 100755 index 00000000000..86398f881be --- /dev/null +++ b/extras/hs-test/script/build-images.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +# Build script for all Docker images based on the common base image + +set -e + +# Get default architecture for multi-arch builds +ARCH=${OS_ARCH:-$(dpkg --print-architecture)} + +# Set up buildx configuration +DOCKER_BUILD_DIR="/scratch/docker-build" +DOCKER_CACHE_DIR="${DOCKER_BUILD_DIR}/docker_cache" +DOCKER_HST_BUILDER="hst_builder" + +if [ -d "${DOCKER_BUILD_DIR}" ] ; then + mkdir -p "${DOCKER_CACHE_DIR}" + + # Create buildx builder if it doesn't exist + if ! docker buildx ls --format "{{.Name}}" | grep -q "${DOCKER_HST_BUILDER}"; then + docker buildx create --use \ + --driver-opt env.http_proxy="$HTTP_PROXY" \ + --driver-opt env.https_proxy="$HTTP_PROXY" \ + --driver-opt '"env.no_proxy='"$NO_PROXY"'"' \ + --name=${DOCKER_HST_BUILDER} \ + --driver=docker-container \ + --use --bootstrap || true + fi + + DOCKER_CACHE_ARGS="--builder=${DOCKER_HST_BUILDER} --load --cache-to type=local,dest=${DOCKER_CACHE_DIR},mode=max --cache-from type=local,src=${DOCKER_CACHE_DIR}" +fi + +# Set the tag for the base image +BASE_TAG=${BASE_TAG:-"localhost:5000/vpp-test-base:latest"} + +echo "=== Building base image ===" +# shellcheck disable=2086 +docker buildx build ${DOCKER_CACHE_ARGS} \ + --build-arg UBUNTU_VERSION="${UBUNTU_VERSION:-22.04}" \ + --build-arg http_proxy="$HTTP_PROXY" \ + --build-arg https_proxy="$HTTP_PROXY" \ + --build-arg HTTP_PROXY="$HTTP_PROXY" \ + --build-arg HTTPS_PROXY="$HTTP_PROXY" \ + -t $BASE_TAG -f docker/Dockerfile.base . || { + echo "Error: Failed to build base image" + exit 1 +} + +# Push the base image to the local registry +docker push $BASE_TAG || { + echo "Error: Failed to push base image to local registry" + exit 1 +} + +# Function to build each image +build_image() { + local dockerfile="docker/$1" + local tag=$2 + local add_args="${3:-}" + + if [ ! -f "$dockerfile" ]; then + echo "Warning: Dockerfile $dockerfile doesn't exist, skipping" + return 0 + fi + + echo "=== Building $tag from $dockerfile ===" + echo "Building with architecture: $ARCH" + + # Check if the necessary files for VPP-based images are available + if [[ "$dockerfile" == *"vpp"* || "$dockerfile" == *"nginx"* || "$dockerfile" == *"vcl"* ]]; then + # Check for essential VPP files + for file in vpp-data/bin/vpp vpp-data/lib/*.so; do + if [ ! -e "$file" ]; then + echo "Warning: Required VPP file $file doesn't exist." + fi + done + fi + + # Build the image + # shellcheck disable=2086 + docker build \ + --build-arg UBUNTU_VERSION="${UBUNTU_VERSION:-22.04}" \ + --build-arg OS_ARCH="$ARCH" \ + --build-arg http_proxy="$HTTP_PROXY" \ + --build-arg https_proxy="$HTTP_PROXY" \ + --build-arg HTTP_PROXY="$HTTP_PROXY" \ + --build-arg HTTPS_PROXY="$HTTP_PROXY" \ + $add_args \ + -t "$tag" \ + -f "$dockerfile" . || { + echo "Error: Failed to build $tag" + return 1 + } + + echo "=== Successfully built and pushed $tag ===" +} + +# Build all standard images +echo "=== Building standard images ===" +build_image "Dockerfile.vpp" "hs-test/vpp" +build_image "Dockerfile.nginx" "hs-test/nginx-ldp" +build_image "Dockerfile.nginx-server" "hs-test/nginx-server" +build_image "Dockerfile.h2load" "hs-test/h2load" +build_image "Dockerfile.curl" "hs-test/curl" +build_image "Dockerfile.ab" "hs-test/ab" +build_image "Dockerfile.wrk" "hs-test/wrk" + +# Build HTTP/3 nginx if available +echo "=== Building HTTP/3 nginx image ===" +build_image "Dockerfile.nginx-http3" "hs-test/nginx-http3" + +# Build envoy separately since it doesn't use our base image +echo "=== Building envoy-test ===" +build_image "Dockerfile.envoy" "hs-test/envoy" + +# make cache directory multi-user friendly if it exists +if [ -d "${DOCKER_CACHE_DIR}" ] ; then + chgrp -R docker "${DOCKER_CACHE_DIR}" 2>/dev/null || true + chmod -R g+rwx "${DOCKER_CACHE_DIR}" 2>/dev/null || true +fi + +# cleanup detached images +images=$(docker images --filter "dangling=true" -q --no-trunc) +if [ -n "$images" ]; then + echo "=== Cleaning up dangling images ===" + # shellcheck disable=SC2086 + docker rmi $images || true +fi + +echo "=== All container images built successfully ===" diff --git a/extras/hs-test/script/build_curl.sh b/extras/hs-test/script/build_curl.sh index 1a61b9fd70a..ec893796054 100755 --- a/extras/hs-test/script/build_curl.sh +++ b/extras/hs-test/script/build_curl.sh @@ -1,5 +1,5 @@ #!/bin/bash - +set -x wget https://github.com/stunnel/static-curl/releases/download/8.5.0/curl-static-"$TARGETARCH"-8.5.0.tar.xz tar -xvf ./curl-static-"$TARGETARCH"-8.5.0.tar.xz cp curl /usr/bin/curl \ No newline at end of file diff --git a/extras/hs-test/script/build_hst.sh b/extras/hs-test/script/build_hst.sh index ab482b09f63..0e7656354f7 100755 --- a/extras/hs-test/script/build_hst.sh +++ b/extras/hs-test/script/build_hst.sh @@ -39,15 +39,16 @@ OS_ARCH="$(uname -m)" DOCKER_BUILD_DIR="/scratch/docker-build" DOCKER_CACHE_DIR="${DOCKER_BUILD_DIR}/docker_cache" -if [ -d "${DOCKER_BUILD_DIR}" ] ; then - mkdir -p "${DOCKER_CACHE_DIR}" - DOCKER_HST_BUILDER="hst_builder" - set -x - if ! docker buildx ls --format "{{.Name}}" | grep -q "${DOCKER_HST_BUILDER}"; then - docker buildx create --use --driver-opt env.http_proxy="$HTTP_PROXY" --driver-opt env.https_proxy="$HTTP_PROXY" --driver-opt '"env.no_proxy='"$NO_PROXY"'"' --name=${DOCKER_HST_BUILDER} --driver=docker-container --use --bootstrap || true +# Set up the local registry before creating containers +echo "=== Setting up local registry ===" +if [ -x "$(dirname "$0")/../docker/setup-local-registry.sh" ]; then + "$(dirname "$0")/../docker/setup-local-registry.sh" +else + echo "Warning: setup-local-registry.sh not found or not executable" + echo "Attempting to create and use local registry at localhost:5000" + if ! docker ps | grep -q "local-registry"; then + docker run -d --restart=always -p 5000:5000 --name local-registry registry:2 fi - set -x - DOCKER_CACHE_ARGS="--builder=${DOCKER_HST_BUILDER} --load --cache-to type=local,dest=${DOCKER_CACHE_DIR},mode=max --cache-from type=local,src=${DOCKER_CACHE_DIR}" fi echo "Taking build objects from ${VPP_BUILD_ROOT}" @@ -74,43 +75,27 @@ if [ "$res" -ne 0 ]; then exit 1 fi -docker_build () { - tag=$1 - dockername=$2 - set -ex - # shellcheck disable=2086 - docker buildx build ${DOCKER_CACHE_ARGS} \ - --build-arg UBUNTU_VERSION \ - --build-arg OS_ARCH="$OS_ARCH" \ - --build-arg http_proxy="$HTTP_PROXY" \ - --build-arg https_proxy="$HTTP_PROXY" \ - --build-arg HTTP_PROXY="$HTTP_PROXY" \ - --build-arg HTTPS_PROXY="$HTTP_PROXY" \ - -t "$tag" -f docker/Dockerfile."$dockername" . - set +ex -} - -docker_build hs-test/vpp vpp -docker_build hs-test/nginx-ldp nginx -docker_build hs-test/nginx-server nginx-server -docker_build hs-test/curl curl -docker_build hs-test/envoy envoy -docker_build hs-test/nginx-http3 nginx-http3 -docker_build hs-test/ab ab -docker_build hs-test/wrk wrk -docker_build hs-test/h2load h2load - -# make it multi-user friendly -if [ -d "${DOCKER_CACHE_DIR}" ] ; then - chgrp -R docker "${DOCKER_CACHE_DIR}" - chmod -R g+rwx "${DOCKER_CACHE_DIR}" -fi - -# cleanup detached images -images=$(docker images --filter "dangling=true" -q --no-trunc) -if [ "$images" != "" ]; then - # shellcheck disable=SC2086 - docker rmi $images +# Use the build-images.sh script to build all containers +echo "=== Building all containers using build-images.sh ===" +( + # Export necessary environment variables for build-images.sh + export BASE_TAG="localhost:5000/vpp-test-base:latest" + export OS_ARCH + export UBUNTU_VERSION + export HTTP_PROXY + export HTTPS_PROXY + export NO_PROXY + export DOCKER_CACHE_DIR="${DOCKER_CACHE_DIR}" + export DOCKER_HST_BUILDER="${DOCKER_HST_BUILDER}" + + # Run the build script + ./script/build-images.sh +) + +# Check if the build was successful +if [ $? -ne 0 ]; then + echo "Failed to build Docker images. Check the output above for errors." + exit 1 fi echo "$current_state_hash" > "$LAST_STATE_FILE"