Refactor ccache config to use per-os ccache dirs. 14/31914/6
authorDave Wallace <dwallacelf@gmail.com>
Wed, 7 Apr 2021 00:41:43 +0000 (20:41 -0400)
committerDave Wallace <dwallacelf@gmail.com>
Wed, 21 Apr 2021 20:51:31 +0000 (16:51 -0400)
- Simplify ccache state output to console log by
  using ccache stats command.
- Bump max size of ccache dir to 10G (centos8 is
  approximately 5.7G)
- Clean up extraneous ccache dir after docker build
  is complete.

Change-Id: Ia2b55b7cd47212fce91b4567d0cca69d2ce892ed
Signed-off-by: Dave Wallace <dwallacelf@gmail.com>
31 files changed:
docker/scripts/lib_apt.sh
docker/scripts/lib_dnf.sh
docker/scripts/lib_yum.sh
jenkins-config/clouds/nomad/FDIONOMAD/builder-centos7-prod-x86_64.yaml
jenkins-config/clouds/nomad/FDIONOMAD/builder-centos7-sandbox-x86_64.yaml
jenkins-config/clouds/nomad/FDIONOMAD/builder-centos7-test-x86_64.yaml
jenkins-config/clouds/nomad/FDIONOMAD/builder-centos8-prod-aarch64.yaml
jenkins-config/clouds/nomad/FDIONOMAD/builder-centos8-prod-x86_64.yaml
jenkins-config/clouds/nomad/FDIONOMAD/builder-centos8-sandbox-aarch64.yaml
jenkins-config/clouds/nomad/FDIONOMAD/builder-centos8-sandbox-x86_64.yaml
jenkins-config/clouds/nomad/FDIONOMAD/builder-centos8-test-aarch64.yaml
jenkins-config/clouds/nomad/FDIONOMAD/builder-centos8-test-x86_64.yaml
jenkins-config/clouds/nomad/FDIONOMAD/builder-debian10-prod-x86_64.yaml
jenkins-config/clouds/nomad/FDIONOMAD/builder-debian10-sandbox-x86_64.yaml
jenkins-config/clouds/nomad/FDIONOMAD/builder-debian10-test-x86_64.yaml
jenkins-config/clouds/nomad/FDIONOMAD/builder-debian9-prod-x86_64.yaml
jenkins-config/clouds/nomad/FDIONOMAD/builder-debian9-sandbox-x86_64.yaml
jenkins-config/clouds/nomad/FDIONOMAD/builder-debian9-test-x86_64.yaml
jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu1804-prod-aarch64.yaml
jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu1804-prod-x86_64.yaml
jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu1804-sandbox-aarch64.yaml
jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu1804-sandbox-x86_64.yaml
jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu1804-test-aarch64.yaml
jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu1804-test-x86_64.yaml
jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2004-prod-aarch64.yaml
jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2004-prod-x86_64.yaml
jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2004-sandbox-aarch64.yaml
jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2004-sandbox-x86_64.yaml
jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2004-test-aarch64.yaml
jenkins-config/clouds/nomad/FDIONOMAD/builder-ubuntu2004-test-x86_64.yaml
jjb/scripts/setup_executor_env.sh

index a2e5c98..725aedc 100644 (file)
@@ -221,7 +221,7 @@ generate_apt_dockerfile_clean() {
 
 # Clean up copy-in build tree
 RUN dbld_dump_build_logs.sh \\
-  && rm -rf "/tmp/*" "$DOCKER_BUILD_FILES_DIR"
+  && rm -rf "/tmp/*" "$DOCKER_BUILD_FILES_DIR" "/root/.ccache"
 EOF
 }
 
@@ -260,6 +260,7 @@ WORKDIR /
 $vpp_install_skip_sysctl_envvar
 ENV VPP_ZOMBIE_NOCHECK="1"
 ENV CCACHE_DIR="/scratch/ccache"
+ENV CCACHE_MAXSIZE="10G"
 EOF
     generate_apt_dockerfile_clean
 }
index 34bc40c..4520421 100644 (file)
@@ -38,7 +38,7 @@ generate_dnf_dockerfile_clean() {
 
 # Clean up
 RUN dbld_dump_build_logs.sh \\
-  && rm -rf "/tmp/*"
+  && rm -rf "/tmp/*" "/root/.ccache"
 EOF
 }
 
@@ -167,6 +167,7 @@ RUN gem install package_cloud \\
 WORKDIR /
 ENV VPP_ZOMBIE_NOCHECK="1"
 ENV CCACHE_DIR="/scratch/ccache"
+ENV CCACHE_MAXSIZE="10G"
 EOF
     generate_dnf_dockerfile_clean
 }
index 4e165e6..27c297a 100644 (file)
@@ -144,7 +144,7 @@ generate_yum_dockerfile_clean() {
 
 # Clean up
 RUN dbld_dump_build_logs.sh \\
-  && rm -rf "/tmp/*"
+  && rm -rf "/tmp/*" "/root/.ccache"
 EOF
 }
 
@@ -170,6 +170,7 @@ RUN gem install package_cloud \\
 WORKDIR /
 ENV VPP_ZOMBIE_NOCHECK="1"
 ENV CCACHE_DIR="/scratch/ccache"
+ENV CCACHE_MAXSIZE="10G"
 EOF
     generate_yum_dockerfile_clean
 }
index 95b451f..446ca3f 100644 (file)
@@ -12,4 +12,4 @@ templates:
     labels: "builder-centos7-prod-x86_64"
     memory: 14000
     prefix: "builder-centos7-prod-x86_64"
-    hostVolumes: "/tmp/ccache:/scratch/ccache"
+    hostVolumes: "/tmp/ccache.centos7:/scratch/ccache"
index 6a52ded..0b0b4fc 100644 (file)
@@ -12,4 +12,4 @@ templates:
     labels: "builder-centos7-sandbox-x86_64"
     memory: 14000
     prefix: "builder-centos7-sandbox-x86_64"
-    hostVolumes: "/tmp/ccache:/scratch/ccache"
+    hostVolumes: "/tmp/ccache.centos7:/scratch/ccache"
index 1b58ba5..633e78f 100644 (file)
@@ -12,4 +12,4 @@ templates:
     labels: "builder-centos7-test-x86_64"
     memory: 14000
     prefix: "builder-centos7-test-x86_64"
-    hostVolumes: "/tmp/ccache:/scratch/ccache"
+    hostVolumes: "/tmp/ccache.centos7:/scratch/ccache"
index b01ee17..e24578d 100644 (file)
@@ -3,4 +3,4 @@ templates:
   - image: "fdiotools/builder-centos8:prod-aarch64"
     labels: "builder-centos8-prod-aarch64"
     prefix: "builder-centos8-prod-aarch64"
-    hostVolumes: "/tmp/ccache:/scratch/ccache"
+    hostVolumes: "/tmp/ccache.centos8:/scratch/ccache"
index efba480..7b59d1b 100644 (file)
@@ -12,4 +12,4 @@ templates:
     labels: "builder-centos8-prod-x86_64"
     memory: 14000
     prefix: "builder-centos8-prod-x86_64"
-    hostVolumes: "/tmp/ccache:/scratch/ccache"
+    hostVolumes: "/tmp/ccache.centos8:/scratch/ccache"
index 6dd19a3..b43bf9c 100644 (file)
@@ -3,4 +3,4 @@ templates:
   - image: "fdiotools/builder-centos8:sandbox-aarch64"
     labels: "builder-centos8-sandbox-aarch64"
     prefix: "builder-centos8-sandbox-aarch64"
-    hostVolumes: "/tmp/ccache:/scratch/ccache"
+    hostVolumes: "/tmp/ccache.centos8:/scratch/ccache"
index a5ec1c1..f91a943 100644 (file)
@@ -12,4 +12,4 @@ templates:
     labels: "builder-centos8-sandbox-x86_64"
     memory: 14000
     prefix: "builder-centos8-sandbox-x86_64"
-    hostVolumes: "/tmp/ccache:/scratch/ccache"
+    hostVolumes: "/tmp/ccache.centos8:/scratch/ccache"
index e64c44a..daf222d 100644 (file)
@@ -3,4 +3,4 @@ templates:
   - image: "fdiotools/builder-centos8:test-aarch64"
     labels: "builder-centos8-test-aarch64"
     prefix: "builder-centos8-test-aarch64"
-    hostVolumes: "/tmp/ccache:/scratch/ccache"
+    hostVolumes: "/tmp/ccache.centos8:/scratch/ccache"
index e6fe1df..ac7200c 100644 (file)
@@ -12,4 +12,4 @@ templates:
     labels: "builder-centos8-test-x86_64"
     memory: 14000
     prefix: "builder-centos8-test-x86_64"
-    hostVolumes: "/tmp/ccache:/scratch/ccache"
+    hostVolumes: "/tmp/ccache.centos8:/scratch/ccache"
index 27433e6..d15a462 100644 (file)
@@ -12,4 +12,4 @@ templates:
     labels: "builder-debian10-prod-x86_64"
     memory: 14000
     prefix: "builder-debian10-prod-x86_64"
-    hostVolumes: "/tmp/ccache:/scratch/ccache"
+    hostVolumes: "/tmp/ccache.debian10:/scratch/ccache"
index 9320036..8e2293a 100644 (file)
@@ -12,4 +12,4 @@ templates:
     labels: "builder-debian10-sandbox-x86_64"
     memory: 14000
     prefix: "builder-debian10-sandbox-x86_64"
-    hostVolumes: "/tmp/ccache:/scratch/ccache"
+    hostVolumes: "/tmp/ccache.debian10:/scratch/ccache"
index e40a09b..d49a0ba 100644 (file)
@@ -12,4 +12,4 @@ templates:
     labels: "builder-debian10-test-x86_64"
     memory: 14000
     prefix: "builder-debian10-test-x86_64"
-    hostVolumes: "/tmp/ccache:/scratch/ccache"
+    hostVolumes: "/tmp/ccache.debian10:/scratch/ccache"
index d4115a5..edb1482 100644 (file)
@@ -12,4 +12,4 @@ templates:
     labels: "builder-debian9-prod-x86_64"
     memory: 14000
     prefix: "builder-debian9-prod-x86_64"
-    hostVolumes: "/tmp/ccache:/scratch/ccache"
+    hostVolumes: "/tmp/ccache.debian9:/scratch/ccache"
index 5a3a6c7..d94dceb 100644 (file)
@@ -12,4 +12,4 @@ templates:
     labels: "builder-debian9-sandbox-x86_64"
     memory: 14000
     prefix: "builder-debian9-sandbox-x86_64"
-    hostVolumes: "/tmp/ccache:/scratch/ccache"
+    hostVolumes: "/tmp/ccache.debian9:/scratch/ccache"
index 057527f..66c0604 100644 (file)
@@ -12,4 +12,4 @@ templates:
     labels: "builder-debian9-test-x86_64"
     memory: 14000
     prefix: "builder-debian9-test-x86_64"
-    hostVolumes: "/tmp/ccache:/scratch/ccache"
+    hostVolumes: "/tmp/ccache.debian9:/scratch/ccache"
index d72e1dd..60bee47 100644 (file)
@@ -3,4 +3,4 @@ templates:
   - image: "fdiotools/builder-ubuntu1804:prod-aarch64"
     labels: "builder-ubuntu1804-prod-aarch64"
     prefix: "builder-ubuntu1804-prod-aarch64"
-    hostVolumes: "/tmp/ccache:/scratch/ccache"
+    hostVolumes: "/tmp/ccache.ubuntu1804:/scratch/ccache"
index b6267be..8bde8f3 100644 (file)
@@ -12,4 +12,4 @@ templates:
     labels: "builder-ubuntu1804-prod-x86_64"
     memory: 14000
     prefix: "builder-ubuntu1804-prod-x86_64"
-    hostVolumes: "/tmp/ccache:/scratch/ccache"
+    hostVolumes: "/tmp/ccache.ubuntu1804:/scratch/ccache"
index f897aa4..af7f2c2 100644 (file)
@@ -3,4 +3,4 @@ templates:
   - image: "fdiotools/builder-ubuntu1804:sandbox-aarch64"
     labels: "builder-ubuntu1804-sandbox-aarch64"
     prefix: "builder-ubuntu1804-sandbox-aarch64"
-    hostVolumes: "/tmp/ccache:/scratch/ccache"
+    hostVolumes: "/tmp/ccache.ubuntu1804:/scratch/ccache"
index 9fd8d0c..58c86ce 100644 (file)
@@ -12,4 +12,4 @@ templates:
     labels: "builder-ubuntu1804-sandbox-x86_64"
     memory: 14000
     prefix: "builder-ubuntu1804-sandbox-x86_64"
-    hostVolumes: "/tmp/ccache:/scratch/ccache"
+    hostVolumes: "/tmp/ccache.ubuntu1804:/scratch/ccache"
index bcb41bb..a1b9011 100644 (file)
@@ -3,4 +3,4 @@ templates:
   - image: "fdiotools/builder-ubuntu1804:test-aarch64"
     labels: "builder-ubuntu1804-test-aarch64"
     prefix: "builder-ubuntu1804-test-aarch64"
-    hostVolumes: "/tmp/ccache:/scratch/ccache"
+    hostVolumes: "/tmp/ccache.ubuntu1804:/scratch/ccache"
index 83691cf..c3ff452 100644 (file)
@@ -12,4 +12,4 @@ templates:
     labels: "builder-ubuntu1804-test-x86_64"
     memory: 14000
     prefix: "builder-ubuntu1804-test-x86_64"
-    hostVolumes: "/tmp/ccache:/scratch/ccache"
+    hostVolumes: "/tmp/ccache.ubuntu1804:/scratch/ccache"
index eee2c64..7d4c78d 100644 (file)
@@ -3,4 +3,4 @@ templates:
   - image: "fdiotools/builder-ubuntu2004:prod-aarch64"
     labels: "builder-ubuntu2004-prod-aarch64"
     prefix: "builder-ubuntu2004-prod-aarch64"
-    hostVolumes: "/tmp/ccache:/scratch/ccache"
+    hostVolumes: "/tmp/ccache.ubuntu2004:/scratch/ccache"
index 4cf8a95..1130558 100644 (file)
@@ -12,4 +12,4 @@ templates:
     labels: "builder-ubuntu2004-prod-x86_64"
     memory: 20000
     prefix: "builder-ubuntu2004-prod-x86_64"
-    hostVolumes: "/tmp/ccache:/scratch/ccache"
+    hostVolumes: "/tmp/ccache.ubuntu2004:/scratch/ccache"
index dd4de72..bfc87f2 100644 (file)
@@ -3,4 +3,4 @@ templates:
   - image: "fdiotools/builder-ubuntu2004:sandbox-aarch64"
     labels: "builder-ubuntu2004-sandbox-aarch64"
     prefix: "builder-ubuntu2004-sandbox-aarch64"
-    hostVolumes: "/tmp/ccache:/scratch/ccache"
+    hostVolumes: "/tmp/ccache.ubuntu2004:/scratch/ccache"
index 3fc8d54..b218c61 100644 (file)
@@ -12,4 +12,4 @@ templates:
     labels: "builder-ubuntu2004-sandbox-x86_64"
     memory: 14000
     prefix: "builder-ubuntu2004-sandbox-x86_64"
-    hostVolumes: "/tmp/ccache:/scratch/ccache"
+    hostVolumes: "/tmp/ccache.ubuntu2004:/scratch/ccache"
index 850d53a..a277684 100644 (file)
@@ -3,4 +3,4 @@ templates:
   - image: "fdiotools/builder-ubuntu2004:test-aarch64"
     labels: "builder-ubuntu2004-test-aarch64"
     prefix: "builder-ubuntu2004-test-aarch64"
-    hostVolumes: "/tmp/ccache:/scratch/ccache"
+    hostVolumes: "/tmp/ccache.ubuntu2004:/scratch/ccache"
index 5a1c326..58f661c 100644 (file)
@@ -12,4 +12,4 @@ templates:
     labels: "builder-ubuntu2004-test-x86_64"
     memory: 14000
     prefix: "builder-ubuntu2004-test-x86_64"
-    hostVolumes: "/tmp/ccache:/scratch/ccache"
+    hostVolumes: "/tmp/ccache.ubuntu2004:/scratch/ccache"
index f9aecfa..a6b8216 100755 (executable)
@@ -61,25 +61,19 @@ echo "$long_line"
 echo "DNS nameserver config in '/etc/resolv.conf':"
 cat /etc/resolv.conf || true
 
-if [ -n "${CCACHE_DIR:-}" ] ; then
-    echo "$long_line"
-    if [ -d "$CCACHE_DIR" ] ; then
-        num_ccache_files="$(find $CCACHE_DIR -type f | wc -l)"
-        ccache_conf="$CCACHE_DIR/ccache.conf"
-        echo "CCACHE_DIR='$CCACHE_DIR' ($num_ccache_files ccache files):"
-        du -sh /tmp/ccache
-        df -h /tmp/ccache
-        ls -l $CCACHE_DIR
-        unset -v CCACHE_DISABLE
-        if [ -f "$ccache_conf" ] ; then
-            echo "Contents of $ccache_conf:"
-            cat $ccache_conf
-        fi
-    else
+echo "$long_line"
+if [ -n "$(which ccache || true)" ] ; then
+    if  [ -z "${CCACHE_DIR:-}" ] || [ ! -d "$CCACHE_DIR" ] ; then
         echo "CCACHE_DIR='$CCACHE_DIR' is missing, disabling CCACHE..."
-        unset -v CCACHE_DIR
         export CCACHE_DISABLE="1"
-        echo "CCACHE_DISABLE='${CCACHE_DISABLE:-}'"
     fi
+    if [ -n "${CCACHE_DISABLE:-}" ] ; then
+        echo "CCACHE_DISABLE = '$CCACHE_DISABLE'"
+    fi
+    echo "ccache statistics:"
+    ccache -s
+else
+    echo "WARNING: ccache is not installed!"
+    export CCACHE_DISABLE="1"
 fi
 echo "$long_line"