resources/tools/presentation/input_data_files.py

   1 # Copyright (c) 2018 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Inputs
  15 Download all data.
  16 """
  17
  18 import re
  19 import logging
  20
  21 from os import rename
  22 from os.path import join
  23 from shutil import move
  24 from zipfile import ZipFile, is_zipfile, BadZipfile
  25 from httplib import responses
  26 from requests import get, codes, RequestException, Timeout, TooManyRedirects, \
  27     HTTPError, ConnectionError
  28
  29 from errors import PresentationError
  30 from utils import execute_command
  31
  32 # Chunk size used for file download
  33 CHUNK_SIZE = 512
  34
  35 # Separator used in file names
  36 SEPARATOR = "__"
  37
  38 REGEX_RELEASE = re.compile(r'(\D*)(\d{4}|master)(\D*)')
  39
  40
  41 def _download_file(url, file_name):
  42     """Download a file with input data.
  43
  44     :param url: URL to the file to download.
  45     :param file_name: Name of file to download.
  46     :type url: str
  47     :type file_name: str
  48     :returns: True if the download was successful, otherwise False.
  49     :rtype: bool
  50     """
  51
  52     success = False
  53     try:
  54         logging.info("      Connecting to '{0}' ...".format(url))
  55
  56         response = get(url, stream=True)
  57         code = response.status_code
  58
  59         logging.info("      {0}: {1}".format(code, responses[code]))
  60
  61         if code != codes["OK"]:
  62             return False
  63
  64         logging.info("      Downloading the file '{0}' to '{1}' ...".
  65                      format(url, file_name))
  66
  67         file_handle = open(file_name, "wb")
  68         for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
  69             if chunk:
  70                 file_handle.write(chunk)
  71         file_handle.close()
  72         success = True
  73     except ConnectionError as err:
  74         logging.error("Not possible to connect to '{0}'.".format(url))
  75         logging.debug(str(err))
  76     except HTTPError as err:
  77         logging.error("Invalid HTTP response from '{0}'.".format(url))
  78         logging.debug(str(err))
  79     except TooManyRedirects as err:
  80         logging.error("Request exceeded the configured number "
  81                       "of maximum re-directions.")
  82         logging.debug(str(err))
  83     except Timeout as err:
  84         logging.error("Request timed out.")
  85         logging.debug(str(err))
  86     except RequestException as err:
  87         logging.error("Unexpected HTTP request exception.")
  88         logging.debug(str(err))
  89     except (IOError, ValueError, KeyError) as err:
  90         logging.error("Download failed.")
  91         logging.debug(str(err))
  92
  93     logging.info("      Download finished.")
  94     return success
  95
  96
  97 def _unzip_file(spec, job, build):
  98     """Unzip downloaded source file.
  99
 100     :param spec: Specification read form the specification file.
 101     :param job: Name of the Jenkins job.
 102     :param build: Information about the build.
 103     :type spec: Specification
 104     :type job: str
 105     :type build: dict
 106     :returns: True if the download was successful, otherwise False.
 107     :rtype: bool
 108     """
 109
 110     data_file = spec.input["extract"]
 111     file_name = build["file-name"]
 112     directory = spec.environment["paths"]["DIR[WORKING,DATA]"]
 113     new_name = "{0}{1}{2}".format(file_name.rsplit('.')[-2],
 114                                   SEPARATOR,
 115                                   data_file.split("/")[-1])
 116     logging.info("      Unzipping: '{0}' from '{1}'.".
 117                  format(data_file, file_name))
 118     try:
 119         with ZipFile(file_name, 'r') as zip_file:
 120             zip_file.extract(data_file, directory)
 121         logging.info("      Moving {0} to {1} ...".
 122                      format(join(directory, data_file), directory))
 123         move(join(directory, data_file), directory)
 124         logging.info("      Renaming the file '{0}' to '{1}'".
 125                      format(join(directory, data_file.split("/")[-1]),
 126                             new_name))
 127         rename(join(directory, data_file.split("/")[-1]),
 128                new_name)
 129         spec.set_input_file_name(job, build["build"],
 130                                  new_name)
 131         return True
 132     except (BadZipfile, RuntimeError) as err:
 133         logging.error("Failed to unzip the file '{0}': {1}.".
 134                       format(file_name, str(err)))
 135         return False
 136     except OSError as err:
 137         logging.error("Failed to rename the file '{0}': {1}.".
 138                       format(data_file, str(err)))
 139         return False
 140
 141
 142 def download_and_unzip_data_file(spec, job, build):
 143     """Download and unzip a source file.
 144
 145     :param spec: Specification read form the specification file.
 146     :param job: Name of the Jenkins job.
 147     :param build: Information about the build.
 148     :type spec: Specification
 149     :type job: str
 150     :type build: dict
 151     :returns: True if the download was successful, otherwise False.
 152     :rtype: bool
 153     """
 154
 155     if job.startswith("csit-"):
 156         if spec.input["file-name"].endswith(".zip"):
 157             url = spec.environment["urls"]["URL[JENKINS,CSIT]"]
 158         elif spec.input["file-name"].endswith(".gz"):
 159             url = spec.environment["urls"]["URL[NEXUS,LOG]"]
 160         else:
 161             logging.error("Not supported file format.")
 162             return False
 163     elif job.startswith("hc2vpp-"):
 164         url = spec.environment["urls"]["URL[JENKINS,HC]"]
 165     else:
 166         raise PresentationError("No url defined for the job '{}'.".
 167                                 format(job))
 168     file_name = spec.input["file-name"]
 169     full_name = spec.input["download-path"]. \
 170         format(job=job, build=build["build"], filename=file_name)
 171     url = "{0}/{1}".format(url, full_name)
 172     new_name = join(spec.environment["paths"]["DIR[WORKING,DATA]"],
 173                     "{job}{sep}{build}{sep}{name}".
 174                     format(job=job, sep=SEPARATOR, build=build["build"],
 175                            name=file_name))
 176     # Download the file from the defined source (Jenkins, logs.fd.io):
 177     success = _download_file(url, new_name)
 178
 179     # If not successful, download from docs.fd.io:
 180     if not success:
 181         logging.info("      Trying to download from https://docs.fd.io:")
 182         release = re.search(REGEX_RELEASE, job).group(2)
 183         nexus_file_name = "{job}{sep}{build}{sep}{name}". \
 184             format(job=job, sep=SEPARATOR, build=build["build"], name=file_name)
 185         try:
 186             release = "rls{0}".format(int(release))
 187         except ValueError:
 188             pass
 189         url = "{url}/{release}/{dir}/{file}". \
 190             format(url=spec.environment["urls"]["URL[NEXUS]"],
 191                    release=release,
 192                    dir=spec.environment["urls"]["DIR[NEXUS]"],
 193                    file=nexus_file_name)
 194         success = _download_file(url, new_name)
 195
 196     if success:
 197         spec.set_input_file_name(job, build["build"], new_name)
 198     else:
 199         return False
 200
 201     if spec.input["file-name"].endswith(".gz"):
 202         if "docs.fd.io" in url:
 203             execute_command("gzip --decompress --keep --force {0}".
 204                             format(new_name))
 205         else:
 206             rename(new_name, new_name[:-3])
 207             execute_command("gzip --keep {0}".format(new_name[:-3]))
 208         spec.set_input_file_name(job, build["build"], new_name[:-3])
 209
 210     if new_name.endswith(".zip"):
 211         if is_zipfile(new_name):
 212             return _unzip_file(spec, job, build)
 213         else:
 214             return False
 215     else:
 216         return True