From df0263faee6171805e8bc579a4e0a786e63c3a8d Mon Sep 17 00:00:00 2001 From: Tibor Frank Date: Thu, 4 Mar 2021 09:52:20 +0100 Subject: [PATCH] PAL: Add s3_storage as the data source Priorities for download: 1. s3_storage 2. logs.fd.io 3. docs.fd.io release 4. docs.fd.io master 5. jenkins Change-Id: I7114715c3ec9a33acc8edf3c5c473f4eaffa639d Signed-off-by: Tibor Frank --- resources/tools/presentation/doc/pal_lld.rst | 5 ++++- resources/tools/presentation/input_data_files.py | 26 +++++++++++++++++----- resources/tools/presentation/specification.yaml | 2 +- .../tools/presentation/specification_CPTA.yaml | 1 + .../tools/presentation/specification_local.yaml | 2 +- 5 files changed, 28 insertions(+), 8 deletions(-) diff --git a/resources/tools/presentation/doc/pal_lld.rst b/resources/tools/presentation/doc/pal_lld.rst index 9569433760..38ab6d9bcb 100644 --- a/resources/tools/presentation/doc/pal_lld.rst +++ b/resources/tools/presentation/doc/pal_lld.rst @@ -187,7 +187,10 @@ The structure of the section "Environment" is as follows (example): urls: URL[JENKINS,CSIT]: "https://jenkins.fd.io/view/csit/job" - URL[JENKINS,HC]: "https://jenkins.fd.io/view/hc2vpp/job" + URL[S3_STORAGE,LOG]: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1" + URL[NEXUS,LOG]: "https://logs.fd.io/production/vex-yul-rot-jenkins-1" + URL[NEXUS,DOC]: "https://docs.fd.io/csit" + DIR[NEXUS,DOC]: "report/_static/archive" make-dirs: # List the directories which are created while preparing the environment. diff --git a/resources/tools/presentation/input_data_files.py b/resources/tools/presentation/input_data_files.py index 8b941f2f94..47249b1323 100644 --- a/resources/tools/presentation/input_data_files.py +++ b/resources/tools/presentation/input_data_files.py @@ -200,11 +200,10 @@ def download_and_unzip_data_file(spec, job, build, pid): :rtype: bool """ - # Try to download .gz from logs.fd.io - + # Try to download .gz from s3_storage file_name = spec.input[u"file-name"] url = u"{0}/{1}".format( - spec.environment[u'urls'][u'URL[NEXUS,LOG]'], + spec.environment[u'urls'][u'URL[S3_STORAGE,LOG]'], spec.input[u'download-path'].format( job=job, build=build[u'build'], filename=file_name ) @@ -219,6 +218,25 @@ def download_and_unzip_data_file(spec, job, build, pid): arch = bool(spec.configuration.get(u"archive-inputs", True)) success, downloaded_name = _download_file(url, new_name, arch=arch) + if not success: + # Try to download .gz from logs.fd.io + file_name = spec.input[u"file-name"] + url = u"{0}/{1}".format( + spec.environment[u'urls'][u'URL[NEXUS,LOG]'], + spec.input[u'download-path'].format( + job=job, build=build[u'build'], filename=file_name + ) + ) + new_name = join( + spec.environment[u"paths"][u"DIR[WORKING,DATA]"], + f"{job}{SEPARATOR}{build[u'build']}{SEPARATOR}{file_name}" + ) + + logging.info(f"Trying to download {url}") + + arch = bool(spec.configuration.get(u"archive-inputs", True)) + success, downloaded_name = _download_file(url, new_name, arch=arch) + if not success: # Try to download .gz or .zip from docs.fd.io @@ -260,8 +278,6 @@ def download_and_unzip_data_file(spec, job, build, pid): download_path = spec.input[u"zip-download-path"] if job.startswith(u"csit-"): url = spec.environment[u"urls"][u"URL[JENKINS,CSIT]"] - elif job.startswith(u"hc2vpp-"): - url = spec.environment[u"urls"][u"URL[JENKINS,HC]"] else: raise PresentationError(f"No url defined for the job {job}.") diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml index 72afd35297..5acef6502d 100644 --- a/resources/tools/presentation/specification.yaml +++ b/resources/tools/presentation/specification.yaml @@ -104,7 +104,7 @@ urls: URL[JENKINS,CSIT]: "https://jenkins.fd.io/view/csit/job" - URL[JENKINS,HC]: "https://jenkins.fd.io/view/hc2vpp/job" + URL[S3_STORAGE,LOG]: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1" URL[NEXUS,LOG]: "https://logs.fd.io/production/vex-yul-rot-jenkins-1" URL[NEXUS,DOC]: "https://docs.fd.io/csit" DIR[NEXUS,DOC]: "report/_static/archive" diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml index 50cedf4fbc..96584923e8 100644 --- a/resources/tools/presentation/specification_CPTA.yaml +++ b/resources/tools/presentation/specification_CPTA.yaml @@ -44,6 +44,7 @@ urls: URL[JENKINS,CSIT]: "https://jenkins.fd.io/view/csit/job" + URL[S3_STORAGE,LOG]: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1" URL[NEXUS,LOG]: "https://logs.fd.io/production/vex-yul-rot-jenkins-1" URL[NEXUS,DOC]: "https://docs.fd.io/csit" DIR[NEXUS,DOC]: "trending/_static/archive" diff --git a/resources/tools/presentation/specification_local.yaml b/resources/tools/presentation/specification_local.yaml index 393228c1a5..95970a2fdf 100644 --- a/resources/tools/presentation/specification_local.yaml +++ b/resources/tools/presentation/specification_local.yaml @@ -106,7 +106,7 @@ urls: # Urls where the input files are downloaded from URL[JENKINS,CSIT]: "https://jenkins.fd.io/view/csit/job" - URL[JENKINS,HC]: "https://jenkins.fd.io/view/hc2vpp/job" + URL[S3_STORAGE,LOG]: "https://logs.nginx.service.consul/vex-yul-rot-jenkins-1" URL[NEXUS,LOG]: "https://logs.fd.io/production/vex-yul-rot-jenkins-1" URL[NEXUS,DOC]: "https://docs.fd.io/csit" DIR[NEXUS,DOC]: "report/_static/archive" -- 2.16.6