X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=resources%2Ftools%2Fpresentation%2Finput_data_files.py;h=7e194785705225bce3d0dbf234a3e1ebdf66a846;hb=0354ec6ac7b0c633a9f4c7007c0cba43bc05acbd;hp=d57d1545eac5a314b81378cbcf67b14c48e3f580;hpb=eecad36d7d2275fa47fbcab40dbcf56108ab0a51;p=csit.git diff --git a/resources/tools/presentation/input_data_files.py b/resources/tools/presentation/input_data_files.py index d57d1545ea..7e19478570 100644 --- a/resources/tools/presentation/input_data_files.py +++ b/resources/tools/presentation/input_data_files.py @@ -15,12 +15,15 @@ Download all data. """ +import re +import gzip import logging from os import rename, remove from os.path import join, getsize from shutil import move from zipfile import ZipFile, is_zipfile, BadZipfile + from httplib import responses from requests import get, codes, RequestException, Timeout, TooManyRedirects, \ HTTPError, ConnectionError @@ -34,6 +37,8 @@ CHUNK_SIZE = 512 # Separator used in file names SEPARATOR = "__" +REGEX_RELEASE = re.compile(r'(\D*)(\d{4})(\D*)') + def download_data_files(spec): """Download all data specified in the specification file in the section @@ -47,7 +52,13 @@ def download_data_files(spec): for job, builds in spec.builds.items(): for build in builds: if job.startswith("csit-"): - url = spec.environment["urls"]["URL[JENKINS,CSIT]"] + if spec.input["file-name"].endswith(".zip"): + url = spec.environment["urls"]["URL[JENKINS,CSIT]"] + elif spec.input["file-name"].endswith(".gz"): + url = spec.environment["urls"]["URL[NEXUS,LOG]"] + else: + logging.error("Not supported file format.") + continue elif job.startswith("hc2vpp-"): url = spec.environment["urls"]["URL[JENKINS,HC]"] else: @@ -59,21 +70,42 @@ def download_data_files(spec): url = "{0}/{1}".format(url, full_name) new_name = join( spec.environment["paths"]["DIR[WORKING,DATA]"], - "{job}{sep}{build}{sep}{name}".format(job=job, sep=SEPARATOR, + "{job}{sep}{build}{sep}{name}".format(job=job, + sep=SEPARATOR, build=build["build"], name=file_name)) - - logging.info("Downloading the file '{0}' to '{1}'.". - format(url, new_name)) + logging.info( + "Downloading the file '{0}' to '{1}' ...".format(url, new_name)) status = "failed" try: response = get(url, stream=True) code = response.status_code if code != codes["OK"]: - logging.error("{0}: {1}".format(code, responses[code])) + logging.warning( + "Jenkins: {0}: {1}.".format(code, responses[code])) + logging.info("Trying to download from Nexus:") spec.set_input_state(job, build["build"], "not found") - continue + if code == codes["not_found"]: + release = re.search(REGEX_RELEASE, job).group(2) + nexus_file_name = "{job}{sep}{build}{sep}{name}".\ + format(job=job, sep=SEPARATOR, build=build["build"], + name=file_name) + url = "{url}/rls{release}/{dir}/{file}".\ + format(url=spec.environment["urls"]["URL[NEXUS]"], + release=release, + dir=spec.environment["urls"]["DIR[NEXUS]"], + file=nexus_file_name) + logging.info("Downloading the file '{0}' to '{1}' ...". + format(url, new_name)) + response = get(url, stream=True) + code = response.status_code + if code != codes["OK"]: + logging.error( + "Nexus: {0}: {1}".format(code, responses[code])) + spec.set_input_state( + job, build["build"], "not found") + continue file_handle = open(new_name, "wb") for chunk in response.iter_content(chunk_size=CHUNK_SIZE): @@ -81,25 +113,37 @@ def download_data_files(spec): file_handle.write(chunk) file_handle.close() - expected_length = None - try: - expected_length = int(response.headers["Content-Length"]) - logging.debug(" Expected file size: {0}B". - format(expected_length)) - except KeyError: - logging.debug(" No information about expected size.") - - real_length = getsize(new_name) - logging.debug(" Downloaded size: {0}B".format(real_length)) - - if expected_length: - if real_length == expected_length: + if spec.input["file-name"].endswith(".zip"): + expected_length = None + try: + expected_length = int(response. + headers["Content-Length"]) + logging.debug(" Expected file size: {0}B". + format(expected_length)) + except KeyError: + logging.debug(" No information about expected size.") + + real_length = getsize(new_name) + logging.debug(" Downloaded size: {0}B".format(real_length)) + + if expected_length: + if real_length == expected_length: + status = "downloaded" + logging.info("{0}: {1}".format(code, + responses[code])) + else: + logging.error("The file size differs from the " + "expected size.") + else: status = "downloaded" logging.info("{0}: {1}".format(code, responses[code])) - else: - logging.error("The file size differs from the expected " - "size.") - else: + + elif spec.input["file-name"].endswith(".gz"): + rename(new_name, new_name[:-7]) + with open(new_name[:-7], 'r') as xml_file: + with gzip.open(new_name, 'wb') as gz_file: + gz_file.write(xml_file.read()) + new_name = new_name[:-7] status = "downloaded" logging.info("{0}: {1}".format(code, responses[code])) @@ -160,29 +204,30 @@ def unzip_files(spec): directory = spec.environment["paths"]["DIR[WORKING,DATA]"] file_name = join(build["file-name"]) - if build["status"] == "downloaded" and is_zipfile(file_name): + if build["status"] == "downloaded": logging.info("Unziping: '{0}' from '{1}'.". format(data_file, file_name)) new_name = "{0}{1}{2}".format(file_name.rsplit('.')[-2], SEPARATOR, data_file.split("/")[-1]) try: - with ZipFile(file_name, 'r') as zip_file: - zip_file.extract(data_file, directory) - logging.info("Moving {0} to {1} ...". - format(join(directory, data_file), - directory)) - move(join(directory, data_file), directory) - logging.info("Renaming the file '{0}' to '{1}'". - format(join(directory, - data_file.split("/")[-1]), - new_name)) - rename(join(directory, data_file.split("/")[-1]), - new_name) + if is_zipfile(file_name): + with ZipFile(file_name, 'r') as zip_file: + zip_file.extract(data_file, directory) + logging.info("Moving {0} to {1} ...". + format(join(directory, data_file), + directory)) + move(join(directory, data_file), directory) + logging.info("Renaming the file '{0}' to '{1}'". + format(join(directory, + data_file.split("/")[-1]), + new_name)) + rename(join(directory, data_file.split("/")[-1]), + new_name) + spec.set_input_file_name(job, build["build"], + new_name) status = "unzipped" spec.set_input_state(job, build["build"], status) - spec.set_input_file_name(job, build["build"], - new_name) except (BadZipfile, RuntimeError) as err: logging.error("Failed to unzip the file '{0}': {1}.". format(file_name, str(err))) @@ -191,8 +236,7 @@ def unzip_files(spec): format(data_file, str(err))) finally: if status == "failed": - spec.set_input_file_name(job, build["build"], - None) + spec.set_input_file_name(job, build["build"], None) else: raise PresentationError("The file '{0}' does not exist or " "it is not a zip file".