resources/tools/presentation/input_data_files.py

   1 # Copyright (c) 2017 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Inputs
  15 Download all data.
  16 """
  17
  18 import re
  19 import gzip
  20 import logging
  21
  22 from os import rename, remove
  23 from os.path import join, getsize
  24 from shutil import move
  25 from zipfile import ZipFile, is_zipfile, BadZipfile
  26
  27 from httplib import responses
  28 from requests import get, codes, RequestException, Timeout, TooManyRedirects, \
  29     HTTPError, ConnectionError
  30
  31 from errors import PresentationError
  32
  33
  34 # Chunk size used for file download
  35 CHUNK_SIZE = 512
  36
  37 # Separator used in file names
  38 SEPARATOR = "__"
  39
  40 REGEX_RELEASE = re.compile(r'(\D*)(\d{4}|master)(\D*)')
  41
  42
  43 def download_data_files(spec):
  44     """Download all data specified in the specification file in the section
  45     type: input --> builds.
  46
  47     :param spec: Specification.
  48     :type spec: Specification
  49     :raises: PresentationError if there is no url defined for the job.
  50     """
  51
  52     for job, builds in spec.builds.items():
  53         for build in builds:
  54             if job.startswith("csit-"):
  55                 if spec.input["file-name"].endswith(".zip"):
  56                     url = spec.environment["urls"]["URL[JENKINS,CSIT]"]
  57                 elif spec.input["file-name"].endswith(".gz"):
  58                     url = spec.environment["urls"]["URL[NEXUS,LOG]"]
  59                 else:
  60                     logging.error("Not supported file format.")
  61                     continue
  62             elif job.startswith("hc2vpp-"):
  63                 url = spec.environment["urls"]["URL[JENKINS,HC]"]
  64             else:
  65                 raise PresentationError("No url defined for the job '{}'.".
  66                                         format(job))
  67             file_name = spec.input["file-name"]
  68             full_name = spec.input["download-path"].\
  69                 format(job=job, build=build["build"], filename=file_name)
  70             url = "{0}/{1}".format(url, full_name)
  71             new_name = join(
  72                 spec.environment["paths"]["DIR[WORKING,DATA]"],
  73                 "{job}{sep}{build}{sep}{name}".format(job=job,
  74                                                       sep=SEPARATOR,
  75                                                       build=build["build"],
  76                                                       name=file_name))
  77             logging.info(
  78                 "Downloading the file '{0}' to '{1}' ...".format(url, new_name))
  79
  80             status = "failed"
  81             try:
  82                 response = get(url, stream=True)
  83                 code = response.status_code
  84                 if code != codes["OK"]:
  85                     logging.warning(
  86                         "Jenkins: {0}: {1}.".format(code, responses[code]))
  87                     logging.info("Trying to download from Nexus:")
  88                     spec.set_input_state(job, build["build"], "not found")
  89                     if code == codes["not_found"]:
  90                         release = re.search(REGEX_RELEASE, job).group(2)
  91                         nexus_file_name = "{job}{sep}{build}{sep}{name}".\
  92                             format(job=job, sep=SEPARATOR, build=build["build"],
  93                                    name=file_name)
  94                         try:
  95                             release = "rls".format(int(release))
  96                         except ValueError:
  97                             pass
  98                         url = "{url}/{release}/{dir}/{file}".\
  99                             format(url=spec.environment["urls"]["URL[NEXUS]"],
 100                                    release=release,
 101                                    dir=spec.environment["urls"]["DIR[NEXUS]"],
 102                                    file=nexus_file_name)
 103                         logging.info("Downloading the file '{0}' to '{1}' ...".
 104                                      format(url, new_name))
 105                         response = get(url, stream=True)
 106                         code = response.status_code
 107                         if code != codes["OK"]:
 108                             logging.error(
 109                                 "Nexus: {0}: {1}".format(code, responses[code]))
 110                             spec.set_input_state(
 111                                 job, build["build"], "not found")
 112                             continue
 113
 114                 file_handle = open(new_name, "wb")
 115                 for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
 116                     if chunk:
 117                         file_handle.write(chunk)
 118                 file_handle.close()
 119
 120                 if spec.input["file-name"].endswith(".zip"):
 121                     expected_length = None
 122                     try:
 123                         expected_length = int(response.
 124                                               headers["Content-Length"])
 125                         logging.debug("  Expected file size: {0}B".
 126                                       format(expected_length))
 127                     except KeyError:
 128                         logging.debug("  No information about expected size.")
 129
 130                     real_length = getsize(new_name)
 131                     logging.debug("  Downloaded size: {0}B".format(real_length))
 132
 133                     if expected_length:
 134                         if real_length == expected_length:
 135                             status = "downloaded"
 136                             logging.info("{0}: {1}".format(code,
 137                                                            responses[code]))
 138                         else:
 139                             logging.error("The file size differs from the "
 140                                           "expected size.")
 141                     else:
 142                         status = "downloaded"
 143                         logging.info("{0}: {1}".format(code, responses[code]))
 144
 145                 elif spec.input["file-name"].endswith(".gz"):
 146                     rename(new_name, new_name[:-3])
 147                     with open(new_name[:-3], 'r') as xml_file:
 148                         with gzip.open(new_name, 'wb') as gz_file:
 149                             gz_file.write(xml_file.read())
 150                     new_name = new_name[:-3]
 151                     status = "downloaded"
 152                     logging.info("{0}: {1}".format(code, responses[code]))
 153
 154             except ConnectionError as err:
 155                 logging.error("Not possible to connect to '{0}'.".format(url))
 156                 logging.debug(err)
 157             except HTTPError as err:
 158                 logging.error("Invalid HTTP response from '{0}'.".format(url))
 159                 logging.debug(err)
 160             except TooManyRedirects as err:
 161                 logging.error("Request exceeded the configured number "
 162                               "of maximum re-directions.")
 163                 logging.debug(err)
 164             except Timeout as err:
 165                 logging.error("Request timed out.")
 166                 logging.debug(err)
 167             except RequestException as err:
 168                 logging.error("Unexpected HTTP request exception.")
 169                 logging.debug(err)
 170             except (IOError, ValueError, KeyError) as err:
 171                 logging.error("Download failed.")
 172                 logging.debug("Reason: {0}".format(err))
 173
 174             spec.set_input_state(job, build["build"], status)
 175             spec.set_input_file_name(job, build["build"], new_name)
 176
 177             if status == "failed":
 178                 logging.info("Removing the file '{0}'".format(new_name))
 179                 try:
 180                     remove(new_name)
 181                 except OSError as err:
 182                     logging.warning(str(err))
 183                 spec.set_input_file_name(job, build["build"], None)
 184
 185     unzip_files(spec)
 186
 187
 188 def unzip_files(spec):
 189     """Unzip downloaded zip files
 190
 191     :param spec: Specification.
 192     :type spec: Specification
 193     :raises: PresentationError if the zip file does not exist or it is not a
 194     zip file.
 195     """
 196
 197     if spec.is_debug:
 198         data_file = spec.debug["extract"]
 199     else:
 200         data_file = spec.input["extract"]
 201
 202     for job, builds in spec.builds.items():
 203         for build in builds:
 204             if build["status"] == "failed" or build["status"] == "not found":
 205                 continue
 206             try:
 207                 status = "failed"
 208                 directory = spec.environment["paths"]["DIR[WORKING,DATA]"]
 209                 file_name = join(build["file-name"])
 210
 211                 if build["status"] == "downloaded":
 212                     logging.info("Unziping: '{0}' from '{1}'.".
 213                                  format(data_file, file_name))
 214                     new_name = "{0}{1}{2}".format(file_name.rsplit('.')[-2],
 215                                                   SEPARATOR,
 216                                                   data_file.split("/")[-1])
 217                     try:
 218                         if is_zipfile(file_name):
 219                             with ZipFile(file_name, 'r') as zip_file:
 220                                 zip_file.extract(data_file, directory)
 221                             logging.info("Moving {0} to {1} ...".
 222                                          format(join(directory, data_file),
 223                                                 directory))
 224                             move(join(directory, data_file), directory)
 225                             logging.info("Renaming the file '{0}' to '{1}'".
 226                                          format(join(directory,
 227                                                      data_file.split("/")[-1]),
 228                                                 new_name))
 229                             rename(join(directory, data_file.split("/")[-1]),
 230                                    new_name)
 231                             spec.set_input_file_name(job, build["build"],
 232                                                      new_name)
 233                         status = "unzipped"
 234                         spec.set_input_state(job, build["build"], status)
 235                     except (BadZipfile, RuntimeError) as err:
 236                         logging.error("Failed to unzip the file '{0}': {1}.".
 237                                       format(file_name, str(err)))
 238                     except OSError as err:
 239                         logging.error("Failed to rename the file '{0}': {1}.".
 240                                       format(data_file, str(err)))
 241                     finally:
 242                         if status == "failed":
 243                             spec.set_input_file_name(job, build["build"], None)
 244                 else:
 245                     raise PresentationError("The file '{0}' does not exist or "
 246                                             "it is not a zip file".
 247                                             format(file_name))
 248
 249                 spec.set_input_state(job, build["build"], status)
 250
 251             except KeyError:
 252                 pass