From 82a13664db8e18e12ccff542e24e5a0dbe2fbce4 Mon Sep 17 00:00:00 2001 From: Tibor Frank Date: Thu, 15 Apr 2021 13:38:01 +0200 Subject: [PATCH] PAL: Add a new download path - "{job}/{build}/{filename}" Change-Id: Ibdd86a708e17df32b4866e058614da5e9814a8f0 Signed-off-by: Tibor Frank --- resources/tools/presentation/input_data_files.py | 56 ++++++++++------------ resources/tools/presentation/specification.yaml | 4 +- .../tools/presentation/specification_CPTA.yaml | 6 ++- .../tools/presentation/specification_local.yaml | 4 +- 4 files changed, 34 insertions(+), 36 deletions(-) diff --git a/resources/tools/presentation/input_data_files.py b/resources/tools/presentation/input_data_files.py index e5b26e910c..d4cc8c64e6 100644 --- a/resources/tools/presentation/input_data_files.py +++ b/resources/tools/presentation/input_data_files.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -212,49 +212,44 @@ def download_and_unzip_data_file(spec, job, build, pid): :rtype: bool """ - # Try to download .gz from s3_storage + success = False + file_name = spec.input[u"file-name"] - url = u"{0}/{1}".format( - spec.environment[u'urls'][u'URL[S3_STORAGE,LOG]'], - spec.input[u'download-path'].format( - job=job, build=build[u'build'], filename=file_name - ) - ) new_name = join( spec.environment[u"paths"][u"DIR[WORKING,DATA]"], f"{job}{SEPARATOR}{build[u'build']}{SEPARATOR}{file_name}" ) - - logging.info(f"Trying to download {url}") - arch = bool(spec.configuration.get(u"archive-inputs", True)) - success, downloaded_name = _download_file( - url, new_name, arch=arch, verify=False, repeat=3 - ) + downloaded_name = u"" - if not success: - # Try to download .gz from logs.fd.io - file_name = spec.input[u"file-name"] + # Try to download .gz from s3_storage + for path in spec.input[u'download-path']: url = u"{0}/{1}".format( - spec.environment[u'urls'][u'URL[NEXUS,LOG]'], - spec.input[u'download-path'].format( - job=job, build=build[u'build'], filename=file_name - ) + spec.environment[u'urls'][u'URL[S3_STORAGE,LOG]'], + path.format(job=job, build=build[u'build'], filename=file_name) ) - new_name = join( - spec.environment[u"paths"][u"DIR[WORKING,DATA]"], - f"{job}{SEPARATOR}{build[u'build']}{SEPARATOR}{file_name}" - ) - logging.info(f"Trying to download {url}") - - arch = bool(spec.configuration.get(u"archive-inputs", True)) success, downloaded_name = _download_file( - url, new_name, arch=arch, verify=True, repeat=3 + url, new_name, arch=arch, verify=False, repeat=3 ) + if success: + break if not success: + # Try to download .gz from logs.fd.io + for path in spec.input[u'download-path']: + url = u"{0}/{1}".format( + spec.environment[u'urls'][u'URL[NEXUS,LOG]'], + path.format(job=job, build=build[u'build'], filename=file_name) + ) + logging.info(f"Trying to download {url}") + success, downloaded_name = _download_file( + url, new_name, arch=arch, verify=True, repeat=3 + ) + if success: + break + if not success: # Try to download .gz or .zip from docs.fd.io file_name = (spec.input[u"file-name"], spec.input[u"zip-file-name"]) release = re.search(REGEX_RELEASE, job).group(2) @@ -288,7 +283,6 @@ def download_and_unzip_data_file(spec, job, build, pid): break if not success: - # Try to download .zip from jenkins.fd.io file_name = spec.input[u"zip-file-name"] download_path = spec.input[u"zip-download-path"] @@ -305,9 +299,7 @@ def download_and_unzip_data_file(spec, job, build, pid): spec.environment[u"paths"][u"DIR[WORKING,DATA]"], f"{job}{SEPARATOR}{build[u'build']}{SEPARATOR}{file_name}" ) - logging.info(f"Downloading {url}") - success, downloaded_name = _download_file(url, new_name) if success and downloaded_name.endswith(u".zip"): diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml index 3723af2620..cdc3158c0e 100644 --- a/resources/tools/presentation/specification.yaml +++ b/resources/tools/presentation/specification.yaml @@ -3151,7 +3151,9 @@ arch-file-format: - ".gz" - ".zip" - download-path: "{job}/{build}/archives/{filename}" + download-path: + - "{job}/{build}/{filename}" + - "{job}/{build}/archives/{filename}" extract: "output.xml" zip-file-name: "robot-plugin.zip" diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml index 816df7831a..1772e29abd 100644 --- a/resources/tools/presentation/specification_CPTA.yaml +++ b/resources/tools/presentation/specification_CPTA.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2020 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -609,7 +609,9 @@ general: file-name: "output_info.xml.gz" file-format: ".gz" - download-path: "{job}/{build}/archives/{filename}" + download-path: + - "{job}/{build}/archives/{filename}" + - "{job}/{build}/{filename}" extract: "output.xml" # Number of days from now to the past. Only files generated in this # time period are used. diff --git a/resources/tools/presentation/specification_local.yaml b/resources/tools/presentation/specification_local.yaml index 95970a2fdf..f67b0d269f 100644 --- a/resources/tools/presentation/specification_local.yaml +++ b/resources/tools/presentation/specification_local.yaml @@ -892,7 +892,9 @@ arch-file-format: - ".gz" - ".zip" - download-path: "{job}/{build}/archives/{filename}" + download-path: + - "{job}/{build}/archives/{filename}" + - "{job}/{build}/{filename}" extract: "output.xml" # Specifies the secondary source - Jenkins, it is used if the source file -- 2.16.6