Avoid ci-management archiving files twice

This is a follow-up to https://gerrit.fd.io/r/c/csit/+/20394

https://gerrit.fd.io/r/20119
has changed the way archival works,
everything should now go to logs.fd.io
(instead of appearing on run page in jenkins.fd.io).

The glob pattern for archiving is quite eager,
doing recursive search. That is good, as it can find
also misplaced useful outputs.
But it also means our usage of copy_archives function
creates two copies of archived directtories,
usually archives/ and archives/archive/.

This change renames copy_archives to move_archives,
with few workarounds to support multiple calls.

I also renamed ARCHIVE_DIR value from $CSIT_DIR/archive
to $CSIT_DIR/archives to make "move" operation look natural.

Finally, download_builds function is being removed,
as after recent improvements to VPP compilation speed
nobody seems to be using it.

Change-Id: I19a429e1dfdfaab7fcf32a9609963b1aebd33c6c
Signed-off-by: Vratko Polak <vrpolak@cisco.com>

diff --git a/resources/libraries/bash/entry/bootstrap_verify_perf.sh b/resources/libraries/bash/entry/bootstrap_verify_perf.sh
index 2098f78..fc33440 100644 (file)
--- a/resources/libraries/bash/entry/bootstrap_verify_perf.sh
+++ b/resources/libraries/bash/entry/bootstrap_verify_perf.sh
@@ -1,4 +1,4 @@
-# Copyright (c) 2019 Cisco and/or its affiliates.
+# Copyright (c) 2020 Cisco and/or its affiliates.

 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:


@@ -48,5 +48,5 @@ select_tags || die
 compose_pybot_arguments || die
 run_pybot || die
 untrap_and_unreserve_testbed || die
 compose_pybot_arguments || die
 run_pybot || die
 untrap_and_unreserve_testbed || die

-copy_archives || die
+move_archives || die

 die_on_pybot_error || die
 die_on_pybot_error || die

diff --git a/resources/libraries/bash/entry/bootstrap_vpp_device.sh b/resources/libraries/bash/entry/bootstrap_vpp_device.sh
index ce887f4..ef7bb08 100755 (executable)
--- a/resources/libraries/bash/entry/bootstrap_vpp_device.sh
+++ b/resources/libraries/bash/entry/bootstrap_vpp_device.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
 #!/usr/bin/env bash
 

-# Copyright (c) 2019 Cisco and/or its affiliates.
+# Copyright (c) 2020 Cisco and/or its affiliates.

 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:


@@ -46,5 +46,5 @@ activate_docker_topology || die
 select_vpp_device_tags || die
 compose_pybot_arguments || die
 run_pybot || die
 select_vpp_device_tags || die
 compose_pybot_arguments || die
 run_pybot || die

-copy_archives || die
+move_archives || die

 die_on_pybot_error || die
 die_on_pybot_error || die

diff --git a/resources/libraries/bash/entry/per_patch_device.sh b/resources/libraries/bash/entry/per_patch_device.sh
index 65577c2..01e4de0 100644 (file)
--- a/resources/libraries/bash/entry/per_patch_device.sh
+++ b/resources/libraries/bash/entry/per_patch_device.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
 #!/usr/bin/env bash
 

-# Copyright (c) 2019 Cisco and/or its affiliates.
+# Copyright (c) 2020 Cisco and/or its affiliates.

 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:


@@ -24,7 +24,7 @@ set -exuo pipefail
 # + Everything needed to build VPP is already installed locally.
 # Consequences:
 # + The following directories (relative to VPP repo) are (re)created:
 # + Everything needed to build VPP is already installed locally.
 # Consequences:
 # + The following directories (relative to VPP repo) are (re)created:

-# ++ csit_current, build_current, archive, csit/archive, csit_download_dir.
+# ++ csit_current, build_current, archives, csit/archives, csit_download_dir.

 
 # TODO: Implement some kind of VPP build caching.
 
 
 # TODO: Implement some kind of VPP build caching.
 


@@ -56,6 +56,6 @@ activate_docker_topology || die
 select_vpp_device_tags || die
 compose_pybot_arguments || die
 run_pybot || die
 select_vpp_device_tags || die
 compose_pybot_arguments || die
 run_pybot || die

-copy_archives || die
+move_archives || die

 archive_test_results "csit_current" || die
 die_on_pybot_error || die
 archive_test_results "csit_current" || die
 die_on_pybot_error || die

diff --git a/resources/libraries/bash/entry/per_patch_perf.sh b/resources/libraries/bash/entry/per_patch_perf.sh
index deae8b0..7b64aea 100644 (file)
--- a/resources/libraries/bash/entry/per_patch_perf.sh
+++ b/resources/libraries/bash/entry/per_patch_perf.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
 #!/usr/bin/env bash
 

-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2020 Cisco and/or its affiliates.

 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:


@@ -27,7 +27,7 @@ set -exuo pipefail
 # + Directories build_root, build and csit are reset during the run.
 # + The following directories (relative to VPP repo) are (re)created:
 # ++ csit_new, csit_parent, build_new, build_parent,
 # + Directories build_root, build and csit are reset during the run.
 # + The following directories (relative to VPP repo) are (re)created:
 # ++ csit_new, csit_parent, build_new, build_parent,

-# ++ archive, csit/archive, csit_download_dir.
+# ++ archives, csit/archives, csit_download_dir.

 
 # TODO: Implement some kind of VPP build caching.
 
 
 # TODO: Implement some kind of VPP build caching.
 


@@ -48,8 +48,6 @@ build_vpp_ubuntu_amd64 "CURRENT" || die
 set_aside_commit_build_artifacts || die
 build_vpp_ubuntu_amd64 "PARENT" || die
 set_aside_parent_build_artifacts || die
 set_aside_commit_build_artifacts || die
 build_vpp_ubuntu_amd64 "PARENT" || die
 set_aside_parent_build_artifacts || die

-## Replace previous 4 lines with this to speed up testing.
-#download_builds "REPLACE_WITH_URL" || die

 initialize_csit_dirs || die
 get_test_code "${1-}" || die
 get_test_tag_string || die
 initialize_csit_dirs || die
 get_test_code "${1-}" || die
 get_test_tag_string || die


@@ -74,7 +72,6 @@ for ((iter=0; iter<iterations; iter++)); do
     select_build "build_current" || die
     check_download_dir || die
     run_pybot || die
     select_build "build_current" || die
     check_download_dir || die
     run_pybot || die

-    copy_archives || die

     archive_parse_test_results "csit_current/${iter}" || die
     die_on_pybot_error || die
     # TODO: Use less heavy way to avoid apt remove failures.
     archive_parse_test_results "csit_current/${iter}" || die
     die_on_pybot_error || die
     # TODO: Use less heavy way to avoid apt remove failures.


@@ -82,7 +79,6 @@ for ((iter=0; iter<iterations; iter++)); do
     select_build "build_parent" || die
     check_download_dir || die
     run_pybot || die
     select_build "build_parent" || die
     check_download_dir || die
     run_pybot || die

-    copy_archives || die

     archive_parse_test_results "csit_parent/${iter}" || die
     die_on_pybot_error || die
 done
     archive_parse_test_results "csit_parent/${iter}" || die
     die_on_pybot_error || die
 done

diff --git a/resources/libraries/bash/function/common.sh b/resources/libraries/bash/function/common.sh
index af59bc4..c0aa5f9 100644 (file)
--- a/resources/libraries/bash/function/common.sh
+++ b/resources/libraries/bash/function/common.sh
@@ -198,7 +198,8 @@ function common_dirs () {
     # - RESOURCES_DIR - Path to existing CSIT subdirectory "resources".
     # - TOOLS_DIR - Path to existing resources subdirectory "tools".
     # - PYTHON_SCRIPTS_DIR - Path to existing tools subdirectory "scripts".
     # - RESOURCES_DIR - Path to existing CSIT subdirectory "resources".
     # - TOOLS_DIR - Path to existing resources subdirectory "tools".
     # - PYTHON_SCRIPTS_DIR - Path to existing tools subdirectory "scripts".

-    # - ARCHIVE_DIR - Path to created CSIT subdirectory "archive".
+    # - ARCHIVE_DIR - Path to created CSIT subdirectory "archives".
+    #   The name is chosen to match what ci-management expects.

     # - DOWNLOAD_DIR - Path to created CSIT subdirectory "download_dir".
     # - GENERATED_DIR - Path to created CSIT subdirectory "generated".
     # Directories created if not present:
     # - DOWNLOAD_DIR - Path to created CSIT subdirectory "download_dir".
     # - GENERATED_DIR - Path to created CSIT subdirectory "generated".
     # Directories created if not present:


@@ -240,7 +241,7 @@ function common_dirs () {
         die "Readlink failed."
     }
 
         die "Readlink failed."
     }
 

-    ARCHIVE_DIR=$(readlink -f "${CSIT_DIR}/archive") || {
+    ARCHIVE_DIR=$(readlink -f "${CSIT_DIR}/archives") || {

         die "Readlink failed."
     }
     mkdir -p "${ARCHIVE_DIR}" || die "Mkdir failed."
         die "Readlink failed."
     }
     mkdir -p "${ARCHIVE_DIR}" || die "Mkdir failed."


@@ -295,31 +296,6 @@ function compose_pybot_arguments () {
 }
 
 
 }
 
 

-function copy_archives () {
-
-    # Create additional archive if workspace variable is set.
-    # This way if script is running in jenkins all will be
-    # automatically archived to logs.fd.io.
-    #
-    # Variables read:
-    # - WORKSPACE - Jenkins workspace, copy only if the value is not empty.
-    #   Can be unset, then it speeds up manual testing.
-    # - ARCHIVE_DIR - Path to directory with content to be copied.
-    # Directories updated:
-    # - ${WORKSPACE}/archives/ - Created if does not exist.
-    #   Content of ${ARCHIVE_DIR}/ is copied here.
-    # Functions called:
-    # - die - Print to stderr and exit.
-
-    set -exuo pipefail
-
-    if [[ -n "${WORKSPACE-}" ]]; then
-        mkdir -p "${WORKSPACE}/archives/" || die "Archives dir create failed."
-        cp -rf "${ARCHIVE_DIR}"/* "${WORKSPACE}/archives" || die "Copy failed."
-    fi
-}
-
-

 function deactivate_docker_topology () {
 
     # Deactivate virtual vpp-device topology by removing containers.
 function deactivate_docker_topology () {
 
     # Deactivate virtual vpp-device topology by removing containers.


@@ -568,6 +544,47 @@ function installed () {
 }
 
 
 }
 
 

+function move_archives () {
+
+    # Move archive directory to top of workspace, if not already there.
+    #
+    # ARCHIVE_DIR is positioned relative to CSIT_DIR,
+    # but in some jobs CSIT_DIR is not same as WORKSPACE
+    # (e.g. under VPP_DIR). To simplify ci-management settings,
+    # we want to move the data to the top. We do not want simple copy,
+    # as ci-management is eager with recursive search.
+    #
+    # As some scripts may call this function multiple times,
+    # the actual implementation use copying and deletion,
+    # so the workspace gets "union" of contents (except overwrites on conflict).
+    # The consequence is empty ARCHIVE_DIR remaining after this call.
+    #
+    # As the source directory is emptied,
+    # the check for dirs being different is essential.
+    #
+    # Variables read:
+    # - WORKSPACE - Jenkins workspace, move only if the value is not empty.
+    #   Can be unset, then it speeds up manual testing.
+    # - ARCHIVE_DIR - Path to directory with content to be moved.
+    # Directories updated:
+    # - ${WORKSPACE}/archives/ - Created if does not exist.
+    #   Content of ${ARCHIVE_DIR}/ is moved.
+    # Functions called:
+    # - die - Print to stderr and exit.
+
+    set -exuo pipefail
+
+    if [[ -n "${WORKSPACE-}" ]]; then
+        target=$(readlink -f "${WORKSPACE}/archives")
+        if [[ "${target}" != "${ARCHIVE_DIR}" ]]; then
+            mkdir -p "${target}" || die "Archives dir create failed."
+            cp -rf "${ARCHIVE_DIR}"/* "${target}" || die "Copy failed."
+            rm -rf "${ARCHIVE_DIR}"/* || die "Delete failed."
+        fi
+    fi
+}
+
+

 function reserve_and_cleanup_testbed () {
 
     # Reserve physical testbed, perform cleanup, register trap to unreserve.
 function reserve_and_cleanup_testbed () {
 
     # Reserve physical testbed, perform cleanup, register trap to unreserve.

diff --git a/resources/libraries/bash/function/per_patch.sh b/resources/libraries/bash/function/per_patch.sh
index 8b97780..43a3f97 100644 (file)
--- a/resources/libraries/bash/function/per_patch.sh
+++ b/resources/libraries/bash/function/per_patch.sh
@@ -128,39 +128,10 @@ function compare_test_results () {
 }
 
 
 }
 
 

-function download_builds () {
-
-    # This is mostly useful only for Sandbox testing, to avoid recompilation.
-    #
-    # Arguments:
-    # - ${1} - URL to download VPP builds from.
-    # Variables read:
-    # - VPP_DIR - Path to WORKSPACE, parent of created directories.
-    # Directories created:
-    # - archive - Ends up empty, not to be confused with ${ARCHIVE_DIR}.
-    # - build_current - Holding built artifacts of the patch under test (PUT).
-    # - built_parent - Holding built artifacts of parent of PUT.
-    # Functions called:
-    # - die - Print to stderr and exit, defined in common.sh
+function initialize_csit_dirs () {

 
     set -exuo pipefail
 
 
     set -exuo pipefail
 

-    cd "${VPP_DIR}" || die "Change directory operation failed."
-    dirs=("build-root" "build_parent" "build_current" "archive" "csit_current")
-    rm -rf ${dirs[@]} || {
-        die "Directory removal failed."
-    }
-    wget -N --progress=dot:giga "${1}" || die "Wget download failed."
-    unzip "archive.zip" || die "Archive extraction failed."
-    mv "archive/build_parent" ./ || die "Move operation failed."
-    mv "archive/build_current" ./ || die "Move operation failed."
-}
-
-
-function initialize_csit_dirs () {
-
-    # This could be in prepare_test, but download_builds also needs this.
-    #

     # Variables read:
     # - VPP_DIR - Path to WORKSPACE, parent of created directories.
     # Directories created:
     # Variables read:
     # - VPP_DIR - Path to WORKSPACE, parent of created directories.
     # Directories created: