resources/tools/presentation/input_data_files.py

   1 # Copyright (c) 2017 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Inputs
  15 Download all data.
  16 """
  17
  18 import re
  19 import gzip
  20 import logging
  21
  22 from os import rename, remove
  23 from os.path import join, getsize
  24 from shutil import move
  25 from zipfile import ZipFile, is_zipfile, BadZipfile
  26
  27 from httplib import responses
  28 from requests import get, codes, RequestException, Timeout, TooManyRedirects, \
  29     HTTPError, ConnectionError
  30
  31 from errors import PresentationError
  32
  33
  34 # Chunk size used for file download
  35 CHUNK_SIZE = 512
  36
  37 # Separator used in file names
  38 SEPARATOR = "__"
  39
  40 REGEX_RELEASE = re.compile(r'(\D*)(\d{4}|master)(\D*)')
  41
  42
  43 def download_data_files(spec):
  44     """Download all data specified in the specification file in the section
  45     type: input --> builds.
  46
  47     :param spec: Specification.
  48     :type spec: Specification
  49     :raises: PresentationError if there is no url defined for the job.
  50     """
  51
  52     for job, builds in spec.builds.items():
  53         for build in builds:
  54             if job.startswith("csit-"):
  55                 if spec.input["file-name"].endswith(".zip"):
  56                     url = spec.environment["urls"]["URL[JENKINS,CSIT]"]
  57                 elif spec.input["file-name"].endswith(".gz"):
  58                     url = spec.environment["urls"]["URL[NEXUS,LOG]"]
  59                 else:
  60                     logging.error("Not supported file format.")
  61                     continue
  62             elif job.startswith("hc2vpp-"):
  63                 url = spec.environment["urls"]["URL[JENKINS,HC]"]
  64             else:
  65                 raise PresentationError("No url defined for the job '{}'.".
  66                                         format(job))
  67             file_name = spec.input["file-name"]
  68             full_name = spec.input["download-path"].\
  69                 format(job=job, build=build["build"], filename=file_name)
  70             url = "{0}/{1}".format(url, full_name)
  71             new_name = join(
  72                 spec.environment["paths"]["DIR[WORKING,DATA]"],
  73                 "{job}{sep}{build}{sep}{name}".format(job=job,
  74                                                       sep=SEPARATOR,
  75                                                       build=build["build"],
  76                                                       name=file_name))
  77             logging.info(
  78                 "Downloading the file '{0}' to '{1}' ...".format(url, new_name))
  79
  80             status = "failed"
  81             try:
  82                 response = get(url, stream=True)
  83                 code = response.status_code
  84
  85                 # temporary workaround, remove when output.log.xml is not needed
  86                 if code != codes["OK"] and \
  87                         spec.input["file-name"].endswith(".gz"):
  88                     url = '.'.join(url.split('.')[:-1]) + ".log.gz"
  89                     response = get(url, stream=True)
  90                     code = response.status_code
  91
  92                 if code != codes["OK"]:
  93                     logging.warning(
  94                         "Jenkins: {0}: {1}.".format(code, responses[code]))
  95                     logging.info("Trying to download from Nexus:")
  96                     spec.set_input_state(job, build["build"], "not found")
  97                     if code == codes["not_found"]:
  98                         release = re.search(REGEX_RELEASE, job).group(2)
  99                         nexus_file_name = "{job}{sep}{build}{sep}{name}".\
 100                             format(job=job, sep=SEPARATOR, build=build["build"],
 101                                    name=file_name)
 102                         try:
 103                             release = "rls".format(int(release))
 104                         except ValueError:
 105                             pass
 106                         url = "{url}/{release}/{dir}/{file}".\
 107                             format(url=spec.environment["urls"]["URL[NEXUS]"],
 108                                    release=release,
 109                                    dir=spec.environment["urls"]["DIR[NEXUS]"],
 110                                    file=nexus_file_name)
 111                         logging.info("Downloading the file '{0}' to '{1}' ...".
 112                                      format(url, new_name))
 113                         response = get(url, stream=True)
 114                         code = response.status_code
 115                         if code != codes["OK"]:
 116                             logging.error(
 117                                 "Nexus: {0}: {1}".format(code, responses[code]))
 118                             spec.set_input_state(
 119                                 job, build["build"], "not found")
 120                             continue
 121
 122                 file_handle = open(new_name, "wb")
 123                 for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
 124                     if chunk:
 125                         file_handle.write(chunk)
 126                 file_handle.close()
 127
 128                 if spec.input["file-name"].endswith(".zip"):
 129                     expected_length = None
 130                     try:
 131                         expected_length = int(response.
 132                                               headers["Content-Length"])
 133                         logging.debug("  Expected file size: {0}B".
 134                                       format(expected_length))
 135                     except KeyError:
 136                         logging.debug("  No information about expected size.")
 137
 138                     real_length = getsize(new_name)
 139                     logging.debug("  Downloaded size: {0}B".format(real_length))
 140
 141                     if expected_length:
 142                         if real_length == expected_length:
 143                             status = "downloaded"
 144                             logging.info("{0}: {1}".format(code,
 145                                                            responses[code]))
 146                         else:
 147                             logging.error("The file size differs from the "
 148                                           "expected size.")
 149                     else:
 150                         status = "downloaded"
 151                         logging.info("{0}: {1}".format(code, responses[code]))
 152
 153                 elif spec.input["file-name"].endswith(".gz"):
 154                     rename(new_name, new_name[:-3])
 155                     with open(new_name[:-3], 'r') as xml_file:
 156                         with gzip.open(new_name, 'w') as gz_file:
 157                             gz_file.write(xml_file.read())
 158                     new_name = new_name[:-3]
 159                     status = "downloaded"
 160                     logging.info("{0}: {1}".format(code, responses[code]))
 161
 162             except ConnectionError as err:
 163                 logging.error("Not possible to connect to '{0}'.".format(url))
 164                 logging.debug(err)
 165             except HTTPError as err:
 166                 logging.error("Invalid HTTP response from '{0}'.".format(url))
 167                 logging.debug(err)
 168             except TooManyRedirects as err:
 169                 logging.error("Request exceeded the configured number "
 170                               "of maximum re-directions.")
 171                 logging.debug(err)
 172             except Timeout as err:
 173                 logging.error("Request timed out.")
 174                 logging.debug(err)
 175             except RequestException as err:
 176                 logging.error("Unexpected HTTP request exception.")
 177                 logging.debug(err)
 178             except (IOError, ValueError, KeyError) as err:
 179                 logging.error("Download failed.")
 180                 logging.debug("Reason: {0}".format(err))
 181
 182             spec.set_input_state(job, build["build"], status)
 183             spec.set_input_file_name(job, build["build"], new_name)
 184
 185             if status == "failed":
 186                 logging.info("Removing the file '{0}'".format(new_name))
 187                 try:
 188                     remove(new_name)
 189                 except OSError as err:
 190                     logging.warning(str(err))
 191                 spec.set_input_file_name(job, build["build"], None)
 192
 193     unzip_files(spec)
 194
 195
 196 def unzip_files(spec):
 197     """Unzip downloaded zip files
 198
 199     :param spec: Specification.
 200     :type spec: Specification
 201     :raises: PresentationError if the zip file does not exist or it is not a
 202     zip file.
 203     """
 204
 205     if spec.is_debug:
 206         data_file = spec.debug["extract"]
 207     else:
 208         data_file = spec.input["extract"]
 209
 210     for job, builds in spec.builds.items():
 211         for build in builds:
 212             if build["status"] == "failed" or build["status"] == "not found":
 213                 continue
 214             try:
 215                 status = "failed"
 216                 directory = spec.environment["paths"]["DIR[WORKING,DATA]"]
 217                 file_name = join(build["file-name"])
 218
 219                 if build["status"] == "downloaded":
 220                     logging.info("Unziping: '{0}' from '{1}'.".
 221                                  format(data_file, file_name))
 222                     new_name = "{0}{1}{2}".format(file_name.rsplit('.')[-2],
 223                                                   SEPARATOR,
 224                                                   data_file.split("/")[-1])
 225                     try:
 226                         if is_zipfile(file_name):
 227                             with ZipFile(file_name, 'r') as zip_file:
 228                                 zip_file.extract(data_file, directory)
 229                             logging.info("Moving {0} to {1} ...".
 230                                          format(join(directory, data_file),
 231                                                 directory))
 232                             move(join(directory, data_file), directory)
 233                             logging.info("Renaming the file '{0}' to '{1}'".
 234                                          format(join(directory,
 235                                                      data_file.split("/")[-1]),
 236                                                 new_name))
 237                             rename(join(directory, data_file.split("/")[-1]),
 238                                    new_name)
 239                             spec.set_input_file_name(job, build["build"],
 240                                                      new_name)
 241                         status = "unzipped"
 242                         spec.set_input_state(job, build["build"], status)
 243                     except (BadZipfile, RuntimeError) as err:
 244                         logging.error("Failed to unzip the file '{0}': {1}.".
 245                                       format(file_name, str(err)))
 246                     except OSError as err:
 247                         logging.error("Failed to rename the file '{0}': {1}.".
 248                                       format(data_file, str(err)))
 249                     finally:
 250                         if status == "failed":
 251                             spec.set_input_file_name(job, build["build"], None)
 252                 else:
 253                     raise PresentationError("The file '{0}' does not exist or "
 254                                             "it is not a zip file".
 255                                             format(file_name))
 256
 257                 spec.set_input_state(job, build["build"], status)
 258
 259             except KeyError:
 260                 pass