PAL: Add s3_storage as the data source

Priorities for download:
1. s3_storage
2. logs.fd.io
3. docs.fd.io release
4. docs.fd.io master
5. jenkins

Change-Id: I7114715c3ec9a33acc8edf3c5c473f4eaffa639d
Signed-off-by: Tibor Frank <tifrank@cisco.com>

diff --git a/resources/tools/presentation/doc/pal_lld.rst b/resources/tools/presentation/doc/pal_lld.rst
index 9569433..38ab6d9 100644 (file)
--- a/resources/tools/presentation/doc/pal_lld.rst
+++ b/resources/tools/presentation/doc/pal_lld.rst
@@ -187,7 +187,10 @@ The structure of the section "Environment" is as follows (example):
 
       urls:
         URL[JENKINS,CSIT]: "https://jenkins.fd.io/view/csit/job"
-        URL[JENKINS,HC]: "https://jenkins.fd.io/view/hc2vpp/job"
+        URL[S3_STORAGE,LOG]: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1"
+        URL[NEXUS,LOG]: "https://logs.fd.io/production/vex-yul-rot-jenkins-1"
+        URL[NEXUS,DOC]: "https://docs.fd.io/csit"
+        DIR[NEXUS,DOC]: "report/_static/archive"
 
       make-dirs:
       # List the directories which are created while preparing the environment.

diff --git a/resources/tools/presentation/input_data_files.py b/resources/tools/presentation/input_data_files.py
index 8b941f2..47249b1 100644 (file)
--- a/resources/tools/presentation/input_data_files.py
+++ b/resources/tools/presentation/input_data_files.py
@@ -200,11 +200,10 @@ def download_and_unzip_data_file(spec, job, build, pid):
     :rtype: bool
     """
 
-    # Try to download .gz from logs.fd.io
-
+    # Try to download .gz from s3_storage
     file_name = spec.input[u"file-name"]
     url = u"{0}/{1}".format(
-        spec.environment[u'urls'][u'URL[NEXUS,LOG]'],
+        spec.environment[u'urls'][u'URL[S3_STORAGE,LOG]'],
         spec.input[u'download-path'].format(
             job=job, build=build[u'build'], filename=file_name
         )
@@ -219,6 +218,25 @@ def download_and_unzip_data_file(spec, job, build, pid):
     arch = bool(spec.configuration.get(u"archive-inputs", True))
     success, downloaded_name = _download_file(url, new_name, arch=arch)
 
+    if not success:
+        # Try to download .gz from logs.fd.io
+        file_name = spec.input[u"file-name"]
+        url = u"{0}/{1}".format(
+            spec.environment[u'urls'][u'URL[NEXUS,LOG]'],
+            spec.input[u'download-path'].format(
+                job=job, build=build[u'build'], filename=file_name
+            )
+        )
+        new_name = join(
+            spec.environment[u"paths"][u"DIR[WORKING,DATA]"],
+            f"{job}{SEPARATOR}{build[u'build']}{SEPARATOR}{file_name}"
+        )
+
+        logging.info(f"Trying to download {url}")
+
+        arch = bool(spec.configuration.get(u"archive-inputs", True))
+        success, downloaded_name = _download_file(url, new_name, arch=arch)
+
     if not success:
 
         # Try to download .gz or .zip from docs.fd.io
@@ -260,8 +278,6 @@ def download_and_unzip_data_file(spec, job, build, pid):
         download_path = spec.input[u"zip-download-path"]
         if job.startswith(u"csit-"):
             url = spec.environment[u"urls"][u"URL[JENKINS,CSIT]"]
-        elif job.startswith(u"hc2vpp-"):
-            url = spec.environment[u"urls"][u"URL[JENKINS,HC]"]
         else:
             raise PresentationError(f"No url defined for the job {job}.")
 

diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml
index 72afd35..5acef65 100644 (file)
--- a/resources/tools/presentation/specification.yaml
+++ b/resources/tools/presentation/specification.yaml
@@ -104,7 +104,7 @@
 
   urls:
     URL[JENKINS,CSIT]: "https://jenkins.fd.io/view/csit/job"
-    URL[JENKINS,HC]: "https://jenkins.fd.io/view/hc2vpp/job"
+    URL[S3_STORAGE,LOG]: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1"
     URL[NEXUS,LOG]: "https://logs.fd.io/production/vex-yul-rot-jenkins-1"
     URL[NEXUS,DOC]: "https://docs.fd.io/csit"
     DIR[NEXUS,DOC]: "report/_static/archive"

diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml
index 50cedf4..9658492 100644 (file)
--- a/resources/tools/presentation/specification_CPTA.yaml
+++ b/resources/tools/presentation/specification_CPTA.yaml
@@ -44,6 +44,7 @@
 
   urls:
     URL[JENKINS,CSIT]: "https://jenkins.fd.io/view/csit/job"
+    URL[S3_STORAGE,LOG]: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1"
     URL[NEXUS,LOG]: "https://logs.fd.io/production/vex-yul-rot-jenkins-1"
     URL[NEXUS,DOC]: "https://docs.fd.io/csit"
     DIR[NEXUS,DOC]: "trending/_static/archive"

diff --git a/resources/tools/presentation/specification_local.yaml b/resources/tools/presentation/specification_local.yaml
index 393228c..95970a2 100644 (file)
--- a/resources/tools/presentation/specification_local.yaml
+++ b/resources/tools/presentation/specification_local.yaml
@@ -106,7 +106,7 @@
   urls:
     # Urls where the input files are downloaded from
     URL[JENKINS,CSIT]: "https://jenkins.fd.io/view/csit/job"
-    URL[JENKINS,HC]: "https://jenkins.fd.io/view/hc2vpp/job"
+    URL[S3_STORAGE,LOG]: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1"
     URL[NEXUS,LOG]: "https://logs.fd.io/production/vex-yul-rot-jenkins-1"
     URL[NEXUS,DOC]: "https://docs.fd.io/csit"
     DIR[NEXUS,DOC]: "report/_static/archive"