PAL: Download .gz first

Change-Id: Ic83a63e5043ea1342b40e5b0aab189cfe24ba95c
Signed-off-by: Tibor Frank <tifrank@cisco.com>

diff --git a/\ b/\
new file mode 100644 (file)
index 0000000..c9cacd6
--- /dev/null
+++ b/\
@@ -0,0 +1,15 @@
+PAL: Download .gz first
+
+Change-Id: Ic83a63e5043ea1342b40e5b0aab189cfe24ba95c
+Signed-off-by: Tibor Frank <tifrank@cisco.com>
+
+# Please enter the commit message for your changes. Lines starting
+# with '#' will be ignored, and an empty message aborts the commit.
+#
+# Date:      Sun Mar 15 18:58:46 2020 +0100
+#
+# On branch report-comparisons
+# Changes to be committed:
+#      modified:   resources/tools/presentation/input_data_files.py
+#      modified:   resources/tools/presentation/specification.yaml
+#

diff --git a/resources/tools/presentation/input_data_files.py b/resources/tools/presentation/input_data_files.py
index b1bb8cd..e1fa5b2 100644 (file)
--- a/resources/tools/presentation/input_data_files.py
+++ b/resources/tools/presentation/input_data_files.py
@@ -17,6 +17,7 @@ Download all data.
 
 import re
 import logging
+import gzip
 
 from os import rename, mkdir
 from os.path import join
@@ -238,37 +239,10 @@ def download_and_unzip_data_file(spec, job, build, pid, log):
 
     if not success:
 
-        # Try to download .gz from docs.fd.io
-
-        file_name = spec.input[u"file-name"]
-        url = u"{0}/{1}".format(
-            spec.environment[u"urls"][u"URL[NEXUS,DOC]"],
-            spec.input[u"download-path"].format(
-                job=job, build=build[u"build"], filename=file_name
-            )
-        )
-        new_name = join(
-            spec.environment[u"paths"][u"DIR[WORKING,DATA]"],
-            f"{job}{SEPARATOR}{build[u'build']}{SEPARATOR}{file_name}"
-        )
-
-        logging.info(f"Downloading {url}")
-
-        success, downloaded_name = _download_file(url, new_name, log, arch=arch)
-
-    if not success:
-
-        # Try to download .zip from docs.fd.io
-
-        file_name = spec.input[u"zip-file-name"]
-        new_name = join(
-            spec.environment[u"paths"][u"DIR[WORKING,DATA]"],
-            f"{job}{SEPARATOR}{build[u'build']}{SEPARATOR}{file_name}"
-        )
+        # Try to download .gz or .zip from docs.fd.io
+        file_name = (spec.input[u"file-name"], spec.input[u"zip-file-name"])
         release = re.search(REGEX_RELEASE, job).group(2)
-        for rls in (release, u"master"):
-            nexus_file_name = \
-                f"{job}{SEPARATOR}{build[u'build']}{SEPARATOR}{file_name}"
+        for idx, rls in enumerate((release, u"master", )):
             try:
                 rls = f"rls{int(rls)}"
             except ValueError:
@@ -278,19 +252,30 @@ def download_and_unzip_data_file(spec, job, build, pid, log):
                 f"{spec.environment[u'urls'][u'URL[NEXUS,DOC]']}/"
                 f"{rls}/"
                 f"{spec.environment[u'urls'][u'DIR[NEXUS,DOC]']}/"
-                f"{nexus_file_name}"
+                f"{job}{SEPARATOR}{build[u'build']}{SEPARATOR}{file_name[idx]}"
             )
 
             logging.info(f"Downloading {url}")
 
-            success, downloaded_name = _download_file(url, new_name, log)
+            new_name = join(
+                spec.environment[u"paths"][u"DIR[WORKING,DATA]"],
+                f"{job}{SEPARATOR}{build[u'build']}{SEPARATOR}{file_name[idx]}"
+            )
+            success, downloaded_name = _download_file(
+                url, new_name, log, arch=arch
+            )
             if success:
+                file_name = file_name[idx]
+                if file_name.endswith(u".gz"):
+                    with gzip.open(downloaded_name[:-3], u"rb") as gzip_file:
+                        file_content = gzip_file.read()
+                    with open(downloaded_name[:-3], u"wb") as xml_file:
+                        xml_file.write(file_content)
                 break
 
     if not success:
 
         # Try to download .zip from jenkins.fd.io
-
         file_name = spec.input[u"zip-file-name"]
         download_path = spec.input[u"zip-download-path"]
         if job.startswith(u"csit-"):

diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml
index 6d65874..0febbc9 100644 (file)
--- a/resources/tools/presentation/specification.yaml
+++ b/resources/tools/presentation/specification.yaml
@@ -3237,7 +3237,7 @@
   reference:
     title: "rls1908 xxv710"
     data: "vpp-performance-changes-2n-skx-ref"
-#    data-replacement: "replacement-1908-vpp-throughput-lat-tsa-2n-skx-vhost"
+    data-replacement: "replacement-1908-vpp-throughput-lat-tsa-2n-skx-vhost"
     nic: "NIC_Intel-XXV710"
   compare:
     title: "rls2001 xxv710"