PAL: Add a new download path

- "{job}/{build}/{filename}"

Change-Id: Ibdd86a708e17df32b4866e058614da5e9814a8f0
Signed-off-by: Tibor Frank <tifrank@cisco.com>

diff --git a/resources/tools/presentation/input_data_files.py b/resources/tools/presentation/input_data_files.py
index e5b26e9..d4cc8c6 100644 (file)
--- a/resources/tools/presentation/input_data_files.py
+++ b/resources/tools/presentation/input_data_files.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2020 Cisco and/or its affiliates.
+# Copyright (c) 2021 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -212,49 +212,44 @@ def download_and_unzip_data_file(spec, job, build, pid):
     :rtype: bool
     """
 
-    # Try to download .gz from s3_storage
+    success = False
+
     file_name = spec.input[u"file-name"]
-    url = u"{0}/{1}".format(
-        spec.environment[u'urls'][u'URL[S3_STORAGE,LOG]'],
-        spec.input[u'download-path'].format(
-            job=job, build=build[u'build'], filename=file_name
-        )
-    )
     new_name = join(
         spec.environment[u"paths"][u"DIR[WORKING,DATA]"],
         f"{job}{SEPARATOR}{build[u'build']}{SEPARATOR}{file_name}"
     )
-
-    logging.info(f"Trying to download {url}")
-
     arch = bool(spec.configuration.get(u"archive-inputs", True))
-    success, downloaded_name = _download_file(
-        url, new_name, arch=arch, verify=False, repeat=3
-    )
+    downloaded_name = u""
 
-    if not success:
-        # Try to download .gz from logs.fd.io
-        file_name = spec.input[u"file-name"]
+    # Try to download .gz from s3_storage
+    for path in spec.input[u'download-path']:
         url = u"{0}/{1}".format(
-            spec.environment[u'urls'][u'URL[NEXUS,LOG]'],
-            spec.input[u'download-path'].format(
-                job=job, build=build[u'build'], filename=file_name
-            )
+            spec.environment[u'urls'][u'URL[S3_STORAGE,LOG]'],
+            path.format(job=job, build=build[u'build'], filename=file_name)
         )
-        new_name = join(
-            spec.environment[u"paths"][u"DIR[WORKING,DATA]"],
-            f"{job}{SEPARATOR}{build[u'build']}{SEPARATOR}{file_name}"
-        )
-
         logging.info(f"Trying to download {url}")
-
-        arch = bool(spec.configuration.get(u"archive-inputs", True))
         success, downloaded_name = _download_file(
-            url, new_name, arch=arch, verify=True, repeat=3
+            url, new_name, arch=arch, verify=False, repeat=3
         )
+        if success:
+            break
 
     if not success:
+        # Try to download .gz from logs.fd.io
+        for path in spec.input[u'download-path']:
+            url = u"{0}/{1}".format(
+                spec.environment[u'urls'][u'URL[NEXUS,LOG]'],
+                path.format(job=job, build=build[u'build'], filename=file_name)
+            )
+            logging.info(f"Trying to download {url}")
+            success, downloaded_name = _download_file(
+                url, new_name, arch=arch, verify=True, repeat=3
+            )
+            if success:
+                break
 
+    if not success:
         # Try to download .gz or .zip from docs.fd.io
         file_name = (spec.input[u"file-name"], spec.input[u"zip-file-name"])
         release = re.search(REGEX_RELEASE, job).group(2)
@@ -288,7 +283,6 @@ def download_and_unzip_data_file(spec, job, build, pid):
                 break
 
     if not success:
-
         # Try to download .zip from jenkins.fd.io
         file_name = spec.input[u"zip-file-name"]
         download_path = spec.input[u"zip-download-path"]
@@ -305,9 +299,7 @@ def download_and_unzip_data_file(spec, job, build, pid):
             spec.environment[u"paths"][u"DIR[WORKING,DATA]"],
             f"{job}{SEPARATOR}{build[u'build']}{SEPARATOR}{file_name}"
         )
-
         logging.info(f"Downloading {url}")
-
         success, downloaded_name = _download_file(url, new_name)
 
     if success and downloaded_name.endswith(u".zip"):

diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml
index 3723af2..cdc3158 100644 (file)
--- a/resources/tools/presentation/specification.yaml
+++ b/resources/tools/presentation/specification.yaml
@@ -3151,7 +3151,9 @@
     arch-file-format:
       - ".gz"
       - ".zip"
-    download-path: "{job}/{build}/archives/{filename}"
+    download-path:
+      - "{job}/{build}/{filename}"
+      - "{job}/{build}/archives/{filename}"
     extract: "output.xml"
 
     zip-file-name: "robot-plugin.zip"

diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml
index 816df78..1772e29 100644 (file)
--- a/resources/tools/presentation/specification_CPTA.yaml
+++ b/resources/tools/presentation/specification_CPTA.yaml
@@ -1,4 +1,4 @@
-# Copyright (c) 2020 Cisco and/or its affiliates.
+# Copyright (c) 2021 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -609,7 +609,9 @@
   general:
     file-name: "output_info.xml.gz"
     file-format: ".gz"
-    download-path: "{job}/{build}/archives/{filename}"
+    download-path:
+      - "{job}/{build}/archives/{filename}"
+      - "{job}/{build}/{filename}"
     extract: "output.xml"
     # Number of days from now to the past. Only files generated in this
     # time period are used.

diff --git a/resources/tools/presentation/specification_local.yaml b/resources/tools/presentation/specification_local.yaml
index 95970a2..f67b0d2 100644 (file)
--- a/resources/tools/presentation/specification_local.yaml
+++ b/resources/tools/presentation/specification_local.yaml
@@ -892,7 +892,9 @@
     arch-file-format:
       - ".gz"
       - ".zip"
-    download-path: "{job}/{build}/archives/{filename}"
+    download-path:
+      - "{job}/{build}/archives/{filename}"
+      - "{job}/{build}/{filename}"
     extract: "output.xml"
 
     # Specifies the secondary source - Jenkins, it is used if the source file