resources/tools/presentation/input_data_files.py

   1 # Copyright (c) 2017 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Inputs
  15 Download all data.
  16 """
  17
  18 import re
  19 import gzip
  20 import logging
  21
  22 from os import rename, remove
  23 from os.path import join, getsize
  24 from shutil import move
  25 from zipfile import ZipFile, is_zipfile, BadZipfile
  26
  27 from httplib import responses
  28 from requests import get, codes, RequestException, Timeout, TooManyRedirects, \
  29     HTTPError, ConnectionError
  30
  31 from errors import PresentationError
  32
  33
  34 # Chunk size used for file download
  35 CHUNK_SIZE = 512
  36
  37 # Separator used in file names
  38 SEPARATOR = "__"
  39
  40 REGEX_RELEASE = re.compile(r'(\D*)(\d{4})(\D*)')
  41
  42
  43 def download_data_files(spec):
  44     """Download all data specified in the specification file in the section
  45     type: input --> builds.
  46
  47     :param spec: Specification.
  48     :type spec: Specification
  49     :raises: PresentationError if there is no url defined for the job.
  50     """
  51
  52     for job, builds in spec.builds.items():
  53         for build in builds:
  54             if job.startswith("csit-"):
  55                 if spec.input["file-name"].endswith(".zip"):
  56                     url = spec.environment["urls"]["URL[JENKINS,CSIT]"]
  57                 elif spec.input["file-name"].endswith(".gz"):
  58                     url = spec.environment["urls"]["URL[NEXUS,LOG]"]
  59                 else:
  60                     logging.error("Not supported file format.")
  61                     continue
  62             elif job.startswith("hc2vpp-"):
  63                 url = spec.environment["urls"]["URL[JENKINS,HC]"]
  64             else:
  65                 raise PresentationError("No url defined for the job '{}'.".
  66                                         format(job))
  67             file_name = spec.input["file-name"]
  68             full_name = spec.input["download-path"].\
  69                 format(job=job, build=build["build"], filename=file_name)
  70             url = "{0}/{1}".format(url, full_name)
  71             new_name = join(
  72                 spec.environment["paths"]["DIR[WORKING,DATA]"],
  73                 "{job}{sep}{build}{sep}{name}".format(job=job,
  74                                                       sep=SEPARATOR,
  75                                                       build=build["build"],
  76                                                       name=file_name))
  77             logging.info(
  78                 "Downloading the file '{0}' to '{1}' ...".format(url, new_name))
  79
  80             status = "failed"
  81             try:
  82                 response = get(url, stream=True)
  83                 code = response.status_code
  84                 if code != codes["OK"]:
  85                     logging.warning(
  86                         "Jenkins: {0}: {1}.".format(code, responses[code]))
  87                     logging.info("Trying to download from Nexus:")
  88                     spec.set_input_state(job, build["build"], "not found")
  89                     if code == codes["not_found"]:
  90                         release = re.search(REGEX_RELEASE, job).group(2)
  91                         nexus_file_name = "{job}{sep}{build}{sep}{name}".\
  92                             format(job=job, sep=SEPARATOR, build=build["build"],
  93                                    name=file_name)
  94                         url = "{url}/rls{release}/{dir}/{file}".\
  95                             format(url=spec.environment["urls"]["URL[NEXUS]"],
  96                                    release=release,
  97                                    dir=spec.environment["urls"]["DIR[NEXUS]"],
  98                                    file=nexus_file_name)
  99                         logging.info("Downloading the file '{0}' to '{1}' ...".
 100                                      format(url, new_name))
 101                         response = get(url, stream=True)
 102                         code = response.status_code
 103                         if code != codes["OK"]:
 104                             logging.error(
 105                                 "Nexus: {0}: {1}".format(code, responses[code]))
 106                             spec.set_input_state(
 107                                 job, build["build"], "not found")
 108                             continue
 109
 110                 file_handle = open(new_name, "wb")
 111                 for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
 112                     if chunk:
 113                         file_handle.write(chunk)
 114                 file_handle.close()
 115
 116                 if spec.input["file-name"].endswith(".zip"):
 117                     expected_length = None
 118                     try:
 119                         expected_length = int(response.
 120                                               headers["Content-Length"])
 121                         logging.debug("  Expected file size: {0}B".
 122                                       format(expected_length))
 123                     except KeyError:
 124                         logging.debug("  No information about expected size.")
 125
 126                     real_length = getsize(new_name)
 127                     logging.debug("  Downloaded size: {0}B".format(real_length))
 128
 129                     if expected_length:
 130                         if real_length == expected_length:
 131                             status = "downloaded"
 132                             logging.info("{0}: {1}".format(code,
 133                                                            responses[code]))
 134                         else:
 135                             logging.error("The file size differs from the "
 136                                           "expected size.")
 137                     else:
 138                         status = "downloaded"
 139                         logging.info("{0}: {1}".format(code, responses[code]))
 140
 141                 elif spec.input["file-name"].endswith(".gz"):
 142                     rename(new_name, new_name[:-7])
 143                     with open(new_name[:-7], 'r') as xml_file:
 144                         with gzip.open(new_name, 'wb') as gz_file:
 145                             gz_file.write(xml_file.read())
 146                     new_name = new_name[:-7]
 147                     status = "downloaded"
 148                     logging.info("{0}: {1}".format(code, responses[code]))
 149
 150             except ConnectionError as err:
 151                 logging.error("Not possible to connect to '{0}'.".format(url))
 152                 logging.debug(err)
 153             except HTTPError as err:
 154                 logging.error("Invalid HTTP response from '{0}'.".format(url))
 155                 logging.debug(err)
 156             except TooManyRedirects as err:
 157                 logging.error("Request exceeded the configured number "
 158                               "of maximum re-directions.")
 159                 logging.debug(err)
 160             except Timeout as err:
 161                 logging.error("Request timed out.")
 162                 logging.debug(err)
 163             except RequestException as err:
 164                 logging.error("Unexpected HTTP request exception.")
 165                 logging.debug(err)
 166             except (IOError, ValueError, KeyError) as err:
 167                 logging.error("Download failed.")
 168                 logging.debug("Reason: {0}".format(err))
 169
 170             spec.set_input_state(job, build["build"], status)
 171             spec.set_input_file_name(job, build["build"], new_name)
 172
 173             if status == "failed":
 174                 logging.info("Removing the file '{0}'".format(new_name))
 175                 try:
 176                     remove(new_name)
 177                 except OSError as err:
 178                     logging.warning(str(err))
 179                 spec.set_input_file_name(job, build["build"], None)
 180
 181     unzip_files(spec)
 182
 183
 184 def unzip_files(spec):
 185     """Unzip downloaded zip files
 186
 187     :param spec: Specification.
 188     :type spec: Specification
 189     :raises: PresentationError if the zip file does not exist or it is not a
 190     zip file.
 191     """
 192
 193     if spec.is_debug:
 194         data_file = spec.debug["extract"]
 195     else:
 196         data_file = spec.input["extract"]
 197
 198     for job, builds in spec.builds.items():
 199         for build in builds:
 200             if build["status"] == "failed" or build["status"] == "not found":
 201                 continue
 202             try:
 203                 status = "failed"
 204                 directory = spec.environment["paths"]["DIR[WORKING,DATA]"]
 205                 file_name = join(build["file-name"])
 206
 207                 if build["status"] == "downloaded":
 208                     logging.info("Unziping: '{0}' from '{1}'.".
 209                                  format(data_file, file_name))
 210                     new_name = "{0}{1}{2}".format(file_name.rsplit('.')[-2],
 211                                                   SEPARATOR,
 212                                                   data_file.split("/")[-1])
 213                     try:
 214                         if is_zipfile(file_name):
 215                             with ZipFile(file_name, 'r') as zip_file:
 216                                 zip_file.extract(data_file, directory)
 217                             logging.info("Moving {0} to {1} ...".
 218                                          format(join(directory, data_file),
 219                                                 directory))
 220                             move(join(directory, data_file), directory)
 221                             logging.info("Renaming the file '{0}' to '{1}'".
 222                                          format(join(directory,
 223                                                      data_file.split("/")[-1]),
 224                                                 new_name))
 225                             rename(join(directory, data_file.split("/")[-1]),
 226                                    new_name)
 227                             spec.set_input_file_name(job, build["build"],
 228                                                      new_name)
 229                         status = "unzipped"
 230                         spec.set_input_state(job, build["build"], status)
 231                     except (BadZipfile, RuntimeError) as err:
 232                         logging.error("Failed to unzip the file '{0}': {1}.".
 233                                       format(file_name, str(err)))
 234                     except OSError as err:
 235                         logging.error("Failed to rename the file '{0}': {1}.".
 236                                       format(data_file, str(err)))
 237                     finally:
 238                         if status == "failed":
 239                             spec.set_input_file_name(job, build["build"], None)
 240                 else:
 241                     raise PresentationError("The file '{0}' does not exist or "
 242                                             "it is not a zip file".
 243                                             format(file_name))
 244
 245                 spec.set_input_state(job, build["build"], status)
 246
 247             except KeyError:
 248                 pass