resources/tools/presentation/input_data_files.py

   1 # Copyright (c) 2017 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Inputs
  15 Download all data.
  16 """
  17
  18 import re
  19 import gzip
  20 import logging
  21
  22 from os import rename, remove
  23 from os.path import join, getsize
  24 from shutil import move
  25 from zipfile import ZipFile, is_zipfile, BadZipfile
  26 from httplib import responses
  27 from requests import get, codes, RequestException, Timeout, TooManyRedirects, \
  28     HTTPError, ConnectionError
  29
  30 from errors import PresentationError
  31 from utils import execute_command
  32
  33 # Chunk size used for file download
  34 CHUNK_SIZE = 512
  35
  36 # Separator used in file names
  37 SEPARATOR = "__"
  38
  39 REGEX_RELEASE = re.compile(r'(\D*)(\d{4}|master)(\D*)')
  40
  41
  42 def download_data_files(spec):
  43     """Download all data specified in the specification file in the section
  44     type: input --> builds.
  45
  46     :param spec: Specification.
  47     :type spec: Specification
  48     :raises: PresentationError if there is no url defined for the job.
  49     """
  50
  51     for job, builds in spec.builds.items():
  52         for build in builds:
  53             if job.startswith("csit-"):
  54                 if spec.input["file-name"].endswith(".zip"):
  55                     url = spec.environment["urls"]["URL[JENKINS,CSIT]"]
  56                 elif spec.input["file-name"].endswith(".gz"):
  57                     url = spec.environment["urls"]["URL[NEXUS,LOG]"]
  58                 else:
  59                     logging.error("Not supported file format.")
  60                     continue
  61             elif job.startswith("hc2vpp-"):
  62                 url = spec.environment["urls"]["URL[JENKINS,HC]"]
  63             else:
  64                 raise PresentationError("No url defined for the job '{}'.".
  65                                         format(job))
  66             file_name = spec.input["file-name"]
  67             full_name = spec.input["download-path"].\
  68                 format(job=job, build=build["build"], filename=file_name)
  69             url = "{0}/{1}".format(url, full_name)
  70             new_name = join(
  71                 spec.environment["paths"]["DIR[WORKING,DATA]"],
  72                 "{job}{sep}{build}{sep}{name}".format(job=job,
  73                                                       sep=SEPARATOR,
  74                                                       build=build["build"],
  75                                                       name=file_name))
  76             logging.info(
  77                 "Downloading the file '{0}' to '{1}' ...".format(url, new_name))
  78
  79             status = "failed"
  80             try:
  81                 response = get(url, stream=True)
  82                 code = response.status_code
  83
  84                 if code != codes["OK"]:
  85                     logging.warning(
  86                         "Jenkins: {0}: {1}.".format(code, responses[code]))
  87                     logging.info("Trying to download from Nexus:")
  88                     spec.set_input_state(job, build["build"], "not found")
  89                     if code == codes["not_found"]:
  90                         release = re.search(REGEX_RELEASE, job).group(2)
  91                         nexus_file_name = "{job}{sep}{build}{sep}{name}".\
  92                             format(job=job, sep=SEPARATOR, build=build["build"],
  93                                    name=file_name)
  94                         try:
  95                             release = "rls".format(int(release))
  96                         except ValueError:
  97                             pass
  98                         url = "{url}/{release}/{dir}/{file}".\
  99                             format(url=spec.environment["urls"]["URL[NEXUS]"],
 100                                    release=release,
 101                                    dir=spec.environment["urls"]["DIR[NEXUS]"],
 102                                    file=nexus_file_name)
 103                         logging.info("Downloading the file '{0}' to '{1}' ...".
 104                                      format(url, new_name))
 105                         response = get(url, stream=True)
 106                         code = response.status_code
 107                         if code != codes["OK"]:
 108                             logging.error(
 109                                 "Nexus: {0}: {1}".format(code, responses[code]))
 110                             spec.set_input_state(
 111                                 job, build["build"], "not found")
 112                             continue
 113
 114                 file_handle = open(new_name, "wb")
 115                 for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
 116                     if chunk:
 117                         file_handle.write(chunk)
 118                 file_handle.close()
 119
 120                 if spec.input["file-name"].endswith(".zip"):
 121                     expected_length = None
 122                     try:
 123                         expected_length = int(response.
 124                                               headers["Content-Length"])
 125                         logging.debug("  Expected file size: {0}B".
 126                                       format(expected_length))
 127                     except KeyError:
 128                         logging.debug("  No information about expected size.")
 129
 130                     real_length = getsize(new_name)
 131                     logging.debug("  Downloaded size: {0}B".format(real_length))
 132
 133                     if expected_length:
 134                         if real_length == expected_length:
 135                             status = "downloaded"
 136                             logging.info("{0}: {1}".format(code,
 137                                                            responses[code]))
 138                         else:
 139                             logging.error("The file size differs from the "
 140                                           "expected size.")
 141                     else:
 142                         status = "downloaded"
 143                         logging.info("{0}: {1}".format(code, responses[code]))
 144
 145                 elif spec.input["file-name"].endswith(".gz"):
 146                     if "docs.fd.io" in url:
 147                         execute_command("gzip --decompress --keep --force {0}".
 148                                         format(new_name))
 149                     else:
 150                         rename(new_name, new_name[:-3])
 151                         execute_command("gzip --keep {0}".format(new_name[:-3]))
 152                     new_name = new_name[:-3]
 153                     status = "downloaded"
 154                     logging.info("{0}: {1}".format(code, responses[code]))
 155
 156             except ConnectionError as err:
 157                 logging.error("Not possible to connect to '{0}'.".format(url))
 158                 logging.debug(err)
 159             except HTTPError as err:
 160                 logging.error("Invalid HTTP response from '{0}'.".format(url))
 161                 logging.debug(err)
 162             except TooManyRedirects as err:
 163                 logging.error("Request exceeded the configured number "
 164                               "of maximum re-directions.")
 165                 logging.debug(err)
 166             except Timeout as err:
 167                 logging.error("Request timed out.")
 168                 logging.debug(err)
 169             except RequestException as err:
 170                 logging.error("Unexpected HTTP request exception.")
 171                 logging.debug(err)
 172             except (IOError, ValueError, KeyError) as err:
 173                 logging.error("Download failed.")
 174                 logging.debug("Reason: {0}".format(err))
 175
 176             spec.set_input_state(job, build["build"], status)
 177             spec.set_input_file_name(job, build["build"], new_name)
 178
 179             if status == "failed":
 180                 logging.info("Removing the file '{0}'".format(new_name))
 181                 try:
 182                     remove(new_name)
 183                 except OSError as err:
 184                     logging.warning(str(err))
 185                 spec.set_input_file_name(job, build["build"], None)
 186
 187     unzip_files(spec)
 188
 189
 190 def unzip_files(spec):
 191     """Unzip downloaded zip files
 192
 193     :param spec: Specification.
 194     :type spec: Specification
 195     :raises: PresentationError if the zip file does not exist or it is not a
 196     zip file.
 197     """
 198
 199     if spec.is_debug:
 200         data_file = spec.debug["extract"]
 201     else:
 202         data_file = spec.input["extract"]
 203
 204     for job, builds in spec.builds.items():
 205         for build in builds:
 206             if build["status"] == "failed" or build["status"] == "not found":
 207                 continue
 208             try:
 209                 status = "failed"
 210                 directory = spec.environment["paths"]["DIR[WORKING,DATA]"]
 211                 file_name = join(build["file-name"])
 212
 213                 if build["status"] == "downloaded":
 214                     logging.info("Unziping: '{0}' from '{1}'.".
 215                                  format(data_file, file_name))
 216                     new_name = "{0}{1}{2}".format(file_name.rsplit('.')[-2],
 217                                                   SEPARATOR,
 218                                                   data_file.split("/")[-1])
 219                     try:
 220                         if is_zipfile(file_name):
 221                             with ZipFile(file_name, 'r') as zip_file:
 222                                 zip_file.extract(data_file, directory)
 223                             logging.info("Moving {0} to {1} ...".
 224                                          format(join(directory, data_file),
 225                                                 directory))
 226                             move(join(directory, data_file), directory)
 227                             logging.info("Renaming the file '{0}' to '{1}'".
 228                                          format(join(directory,
 229                                                      data_file.split("/")[-1]),
 230                                                 new_name))
 231                             rename(join(directory, data_file.split("/")[-1]),
 232                                    new_name)
 233                             spec.set_input_file_name(job, build["build"],
 234                                                      new_name)
 235                         status = "unzipped"
 236                         spec.set_input_state(job, build["build"], status)
 237                     except (BadZipfile, RuntimeError) as err:
 238                         logging.error("Failed to unzip the file '{0}': {1}.".
 239                                       format(file_name, str(err)))
 240                     except OSError as err:
 241                         logging.error("Failed to rename the file '{0}': {1}.".
 242                                       format(data_file, str(err)))
 243                     finally:
 244                         if status == "failed":
 245                             spec.set_input_file_name(job, build["build"], None)
 246                 else:
 247                     raise PresentationError("The file '{0}' does not exist or "
 248                                             "it is not a zip file".
 249                                             format(file_name))
 250
 251                 spec.set_input_state(job, build["build"], status)
 252
 253             except KeyError:
 254                 pass