resources/tools/presentation/input_data_files.py

   1 # Copyright (c) 2018 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Inputs
  15 Download all data.
  16 """
  17
  18 import re
  19
  20 from os import rename, mkdir
  21 from os.path import join
  22 from zipfile import ZipFile, is_zipfile, BadZipfile
  23 from httplib import responses
  24 from requests import get, codes, RequestException, Timeout, TooManyRedirects, \
  25     HTTPError, ConnectionError
  26
  27 from errors import PresentationError
  28 from utils import execute_command
  29
  30 # Chunk size used for file download
  31 CHUNK_SIZE = 512
  32
  33 # Separator used in file names
  34 SEPARATOR = "__"
  35
  36 REGEX_RELEASE = re.compile(r'(\D*)(\d{4}|master)(\D*)')
  37
  38
  39 def _download_file(url, file_name, log):
  40     """Download a file with input data.
  41
  42     :param url: URL to the file to download.
  43     :param file_name: Name of file to download.
  44     :param log: List of log messages.
  45     :type url: str
  46     :type file_name: str
  47     :type log: list of tuples (severity, msg)
  48     :returns: True if the download was successful, otherwise False.
  49     :rtype: bool
  50     """
  51
  52     success = False
  53     try:
  54         log.append(("INFO", "    Connecting to '{0}' ...".format(url)))
  55         response = get(url, stream=True)
  56         code = response.status_code
  57         log.append(("INFO", "    {0}: {1}".format(code, responses[code])))
  58
  59         if code != codes["OK"]:
  60             url = url.replace("_info", "")
  61             log.append(("INFO", "    Connecting to '{0}' ...".format(url)))
  62             response = get(url, stream=True)
  63             code = response.status_code
  64             log.append(("INFO", "    {0}: {1}".format(code, responses[code])))
  65             if code != codes["OK"]:
  66                 return False, file_name
  67             file_name = file_name.replace("_info", "")
  68
  69         log.append(("INFO", "    Downloading the file '{0}' to '{1}' ...".
  70                     format(url, file_name)))
  71
  72         with open(file_name, "wb") as file_handle:
  73             for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
  74                 if chunk:
  75                     file_handle.write(chunk)
  76         success = True
  77     except ConnectionError as err:
  78         log.append(("ERROR", "Not possible to connect to '{0}'.".format(url)))
  79         log.append(("DEBUG", repr(err)))
  80     except HTTPError as err:
  81         log.append(("ERROR", "Invalid HTTP response from '{0}'.".format(url)))
  82         log.append(("DEBUG", repr(err)))
  83     except TooManyRedirects as err:
  84         log.append(("ERROR", "Request exceeded the configured number "
  85                              "of maximum re-directions."))
  86         log.append(("DEBUG", repr(err)))
  87     except Timeout as err:
  88         log.append(("ERROR", "Request timed out."))
  89         log.append(("DEBUG", repr(err)))
  90     except RequestException as err:
  91         log.append(("ERROR", "Unexpected HTTP request exception."))
  92         log.append(("DEBUG", repr(err)))
  93     except (IOError, ValueError, KeyError) as err:
  94         log.append(("ERROR", "Download failed."))
  95         log.append(("DEBUG", repr(err)))
  96
  97     log.append(("INFO", "    Download finished."))
  98     return success, file_name
  99
 100
 101 def _unzip_file(spec, build, pid, log):
 102     """Unzip downloaded source file.
 103
 104     :param spec: Specification read form the specification file.
 105     :param build: Information about the build.
 106     :param log: List of log messages.
 107     :type spec: Specification
 108     :type build: dict
 109     :type log: list of tuples (severity, msg)
 110     :returns: True if the download was successful, otherwise False.
 111     :rtype: bool
 112     """
 113
 114     data_file = spec.input["extract"]
 115     file_name = build["file-name"]
 116     directory = spec.environment["paths"]["DIR[WORKING,DATA]"]
 117     tmp_dir = join(directory, str(pid))
 118     try:
 119         mkdir(tmp_dir)
 120     except OSError:
 121         pass
 122     new_name = "{0}{1}{2}".format(file_name.rsplit('.')[-2],
 123                                   SEPARATOR,
 124                                   data_file.split("/")[-1])
 125
 126     log.append(("INFO", "    Unzipping: '{0}' from '{1}'.".
 127                 format(data_file, file_name)))
 128     try:
 129         with ZipFile(file_name, 'r') as zip_file:
 130             zip_file.extract(data_file, tmp_dir)
 131         log.append(("INFO", "    Renaming the file '{0}' to '{1}'".
 132                     format(join(tmp_dir, data_file), new_name)))
 133         rename(join(tmp_dir, data_file), new_name)
 134         build["file-name"] = new_name
 135         return True
 136     except (BadZipfile, RuntimeError) as err:
 137         log.append(("ERROR", "Failed to unzip the file '{0}': {1}.".
 138                     format(file_name, str(err))))
 139         return False
 140     except OSError as err:
 141         log.append(("ERROR", "Failed to rename the file '{0}': {1}.".
 142                     format(data_file, str(err))))
 143         return False
 144
 145
 146 def download_and_unzip_data_file(spec, job, build, pid, log):
 147     """Download and unzip a source file.
 148
 149     :param spec: Specification read form the specification file.
 150     :param job: Name of the Jenkins job.
 151     :param build: Information about the build.
 152     :param pid: PID of the process executing this method.
 153     :param log: List of log messages.
 154     :type spec: Specification
 155     :type job: str
 156     :type build: dict
 157     :type pid: int
 158     :type log: list of tuples (severity, msg)
 159     :returns: True if the download was successful, otherwise False.
 160     :rtype: bool
 161     """
 162
 163     if job.startswith("csit-"):
 164         if spec.input["file-name"].endswith(".zip"):
 165             url = spec.environment["urls"]["URL[JENKINS,CSIT]"]
 166         elif spec.input["file-name"].endswith(".gz"):
 167             url = spec.environment["urls"]["URL[NEXUS,LOG]"]
 168         else:
 169             log.append(("ERROR", "Not supported file format."))
 170             return False
 171     elif job.startswith("hc2vpp-"):
 172         url = spec.environment["urls"]["URL[JENKINS,HC]"]
 173     elif job.startswith("intel-dnv-"):
 174         url = spec.environment["urls"]["URL[VIRL,DNV]"].format(release=job[-4:])
 175     else:
 176         raise PresentationError("No url defined for the job '{}'.".format(job))
 177     file_name = spec.input["file-name"]
 178     full_name = spec.input["download-path"]. \
 179         format(job=job, build=build["build"], filename=file_name)
 180     if not job.startswith("intel-dnv-"):
 181         url = "{0}/{1}".format(url, full_name)
 182     new_name = join(spec.environment["paths"]["DIR[WORKING,DATA]"],
 183                     "{job}{sep}{build}{sep}{name}".
 184                     format(job=job, sep=SEPARATOR, build=build["build"],
 185                            name=file_name))
 186
 187     # Download the file from the defined source (Jenkins, logs.fd.io):
 188     success, downloaded_name = _download_file(url, new_name, log)
 189     if success:
 190         new_name = downloaded_name
 191
 192     if success and new_name.endswith(".zip"):
 193         if not is_zipfile(new_name):
 194             success = False
 195
 196     # If not successful, download from docs.fd.io:
 197     if not success:
 198         log.append(("INFO", "    Trying to download from https://docs.fd.io:"))
 199         release = re.search(REGEX_RELEASE, job).group(2)
 200         for rls in (release, "master"):
 201             nexus_file_name = "{job}{sep}{build}{sep}{name}". \
 202                 format(job=job, sep=SEPARATOR, build=build["build"],
 203                        name=file_name)
 204             try:
 205                 rls = "rls{0}".format(int(rls))
 206             except ValueError:
 207                 pass
 208             url = "{url}/{release}/{dir}/{file}". \
 209                 format(url=spec.environment["urls"]["URL[NEXUS]"],
 210                        release=rls,
 211                        dir=spec.environment["urls"]["DIR[NEXUS]"],
 212                        file=nexus_file_name)
 213             success, new_name = _download_file(url, new_name, log)
 214             if success:
 215                 break
 216
 217     if success:
 218         build["file-name"] = new_name
 219     else:
 220         return False
 221
 222     if spec.configuration.get("archive-inputs", True):
 223         if spec.input["file-name"].endswith(".gz"):
 224             if "docs.fd.io" in url:
 225                 execute_command("gzip --decompress --keep --force {0}".
 226                                 format(new_name))
 227             else:
 228                 rename(new_name, new_name[:-3])
 229                 execute_command("gzip --keep {0}".format(new_name[:-3]))
 230             build["file-name"] = new_name[:-3]
 231
 232     if new_name.endswith(".zip"):
 233         if is_zipfile(new_name):
 234             return _unzip_file(spec, build, pid, log)
 235         else:
 236             log.append(("ERROR",
 237                         "Zip file '{0}' is corrupted.".format(new_name)))
 238             return False
 239     else:
 240         return True