resources/tools/presentation/input_data_files.py

   1 # Copyright (c) 2017 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Inputs
  15 Download all data.
  16 """
  17
  18 import re
  19 import gzip
  20 import logging
  21
  22 from os import rename, remove
  23 from os.path import join, getsize
  24 from shutil import move
  25 from zipfile import ZipFile, is_zipfile, BadZipfile
  26 from httplib import responses
  27 from requests import get, codes, RequestException, Timeout, TooManyRedirects, \
  28     HTTPError, ConnectionError
  29
  30 from errors import PresentationError
  31 from utils import execute_command
  32
  33 # Chunk size used for file download
  34 CHUNK_SIZE = 512
  35
  36 # Separator used in file names
  37 SEPARATOR = "__"
  38
  39 REGEX_RELEASE = re.compile(r'(\D*)(\d{4}|master)(\D*)')
  40
  41
  42 def download_data_files(spec):
  43     """Download all data specified in the specification file in the section
  44     type: input --> builds.
  45
  46     :param spec: Specification.
  47     :type spec: Specification
  48     :raises: PresentationError if there is no url defined for the job.
  49     """
  50
  51     for job, builds in spec.builds.items():
  52         for build in builds:
  53             if job.startswith("csit-"):
  54                 if spec.input["file-name"].endswith(".zip"):
  55                     url = spec.environment["urls"]["URL[JENKINS,CSIT]"]
  56                 elif spec.input["file-name"].endswith(".gz"):
  57                     url = spec.environment["urls"]["URL[NEXUS,LOG]"]
  58                 else:
  59                     logging.error("Not supported file format.")
  60                     continue
  61             elif job.startswith("hc2vpp-"):
  62                 url = spec.environment["urls"]["URL[JENKINS,HC]"]
  63             else:
  64                 raise PresentationError("No url defined for the job '{}'.".
  65                                         format(job))
  66             file_name = spec.input["file-name"]
  67             full_name = spec.input["download-path"].\
  68                 format(job=job, build=build["build"], filename=file_name)
  69             url = "{0}/{1}".format(url, full_name)
  70             new_name = join(
  71                 spec.environment["paths"]["DIR[WORKING,DATA]"],
  72                 "{job}{sep}{build}{sep}{name}".format(job=job,
  73                                                       sep=SEPARATOR,
  74                                                       build=build["build"],
  75                                                       name=file_name))
  76             logging.info(
  77                 "Downloading the file '{0}' to '{1}' ...".format(url, new_name))
  78
  79             status = "failed"
  80             try:
  81                 response = get(url, stream=True)
  82                 code = response.status_code
  83
  84                 # temporary workaround, remove when output.log.xml is not needed
  85                 if code != codes["OK"] and \
  86                         spec.input["file-name"].endswith(".gz"):
  87                     url = '.'.join(url.split('.')[:-1]) + ".log.gz"
  88                     response = get(url, stream=True)
  89                     code = response.status_code
  90
  91                 if code != codes["OK"]:
  92                     logging.warning(
  93                         "Jenkins: {0}: {1}.".format(code, responses[code]))
  94                     logging.info("Trying to download from Nexus:")
  95                     spec.set_input_state(job, build["build"], "not found")
  96                     if code == codes["not_found"]:
  97                         release = re.search(REGEX_RELEASE, job).group(2)
  98                         nexus_file_name = "{job}{sep}{build}{sep}{name}".\
  99                             format(job=job, sep=SEPARATOR, build=build["build"],
 100                                    name=file_name)
 101                         try:
 102                             release = "rls".format(int(release))
 103                         except ValueError:
 104                             pass
 105                         url = "{url}/{release}/{dir}/{file}".\
 106                             format(url=spec.environment["urls"]["URL[NEXUS]"],
 107                                    release=release,
 108                                    dir=spec.environment["urls"]["DIR[NEXUS]"],
 109                                    file=nexus_file_name)
 110                         logging.info("Downloading the file '{0}' to '{1}' ...".
 111                                      format(url, new_name))
 112                         response = get(url, stream=True)
 113                         code = response.status_code
 114                         if code != codes["OK"]:
 115                             logging.error(
 116                                 "Nexus: {0}: {1}".format(code, responses[code]))
 117                             spec.set_input_state(
 118                                 job, build["build"], "not found")
 119                             continue
 120
 121                 file_handle = open(new_name, "wb")
 122                 for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
 123                     if chunk:
 124                         file_handle.write(chunk)
 125                 file_handle.close()
 126
 127                 if spec.input["file-name"].endswith(".zip"):
 128                     expected_length = None
 129                     try:
 130                         expected_length = int(response.
 131                                               headers["Content-Length"])
 132                         logging.debug("  Expected file size: {0}B".
 133                                       format(expected_length))
 134                     except KeyError:
 135                         logging.debug("  No information about expected size.")
 136
 137                     real_length = getsize(new_name)
 138                     logging.debug("  Downloaded size: {0}B".format(real_length))
 139
 140                     if expected_length:
 141                         if real_length == expected_length:
 142                             status = "downloaded"
 143                             logging.info("{0}: {1}".format(code,
 144                                                            responses[code]))
 145                         else:
 146                             logging.error("The file size differs from the "
 147                                           "expected size.")
 148                     else:
 149                         status = "downloaded"
 150                         logging.info("{0}: {1}".format(code, responses[code]))
 151
 152                 elif spec.input["file-name"].endswith(".gz"):
 153                     rename(new_name, new_name[:-3])
 154                     execute_command("gzip --keep {0}".format(new_name[:-3]))
 155                     new_name = new_name[:-3]
 156                     status = "downloaded"
 157                     logging.info("{0}: {1}".format(code, responses[code]))
 158
 159             except ConnectionError as err:
 160                 logging.error("Not possible to connect to '{0}'.".format(url))
 161                 logging.debug(err)
 162             except HTTPError as err:
 163                 logging.error("Invalid HTTP response from '{0}'.".format(url))
 164                 logging.debug(err)
 165             except TooManyRedirects as err:
 166                 logging.error("Request exceeded the configured number "
 167                               "of maximum re-directions.")
 168                 logging.debug(err)
 169             except Timeout as err:
 170                 logging.error("Request timed out.")
 171                 logging.debug(err)
 172             except RequestException as err:
 173                 logging.error("Unexpected HTTP request exception.")
 174                 logging.debug(err)
 175             except (IOError, ValueError, KeyError) as err:
 176                 logging.error("Download failed.")
 177                 logging.debug("Reason: {0}".format(err))
 178
 179             spec.set_input_state(job, build["build"], status)
 180             spec.set_input_file_name(job, build["build"], new_name)
 181
 182             if status == "failed":
 183                 logging.info("Removing the file '{0}'".format(new_name))
 184                 try:
 185                     remove(new_name)
 186                 except OSError as err:
 187                     logging.warning(str(err))
 188                 spec.set_input_file_name(job, build["build"], None)
 189
 190     unzip_files(spec)
 191
 192
 193 def unzip_files(spec):
 194     """Unzip downloaded zip files
 195
 196     :param spec: Specification.
 197     :type spec: Specification
 198     :raises: PresentationError if the zip file does not exist or it is not a
 199     zip file.
 200     """
 201
 202     if spec.is_debug:
 203         data_file = spec.debug["extract"]
 204     else:
 205         data_file = spec.input["extract"]
 206
 207     for job, builds in spec.builds.items():
 208         for build in builds:
 209             if build["status"] == "failed" or build["status"] == "not found":
 210                 continue
 211             try:
 212                 status = "failed"
 213                 directory = spec.environment["paths"]["DIR[WORKING,DATA]"]
 214                 file_name = join(build["file-name"])
 215
 216                 if build["status"] == "downloaded":
 217                     logging.info("Unziping: '{0}' from '{1}'.".
 218                                  format(data_file, file_name))
 219                     new_name = "{0}{1}{2}".format(file_name.rsplit('.')[-2],
 220                                                   SEPARATOR,
 221                                                   data_file.split("/")[-1])
 222                     try:
 223                         if is_zipfile(file_name):
 224                             with ZipFile(file_name, 'r') as zip_file:
 225                                 zip_file.extract(data_file, directory)
 226                             logging.info("Moving {0} to {1} ...".
 227                                          format(join(directory, data_file),
 228                                                 directory))
 229                             move(join(directory, data_file), directory)
 230                             logging.info("Renaming the file '{0}' to '{1}'".
 231                                          format(join(directory,
 232                                                      data_file.split("/")[-1]),
 233                                                 new_name))
 234                             rename(join(directory, data_file.split("/")[-1]),
 235                                    new_name)
 236                             spec.set_input_file_name(job, build["build"],
 237                                                      new_name)
 238                         status = "unzipped"
 239                         spec.set_input_state(job, build["build"], status)
 240                     except (BadZipfile, RuntimeError) as err:
 241                         logging.error("Failed to unzip the file '{0}': {1}.".
 242                                       format(file_name, str(err)))
 243                     except OSError as err:
 244                         logging.error("Failed to rename the file '{0}': {1}.".
 245                                       format(data_file, str(err)))
 246                     finally:
 247                         if status == "failed":
 248                             spec.set_input_file_name(job, build["build"], None)
 249                 else:
 250                     raise PresentationError("The file '{0}' does not exist or "
 251                                             "it is not a zip file".
 252                                             format(file_name))
 253
 254                 spec.set_input_state(job, build["build"], status)
 255
 256             except KeyError:
 257                 pass