resources/tools/presentation/input_data_files.py

   1 # Copyright (c) 2017 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Inputs
  15 Download all data.
  16 """
  17
  18 import re
  19 import gzip
  20 import logging
  21
  22 from os import rename, remove
  23 from os.path import join, getsize
  24 from shutil import move
  25 from zipfile import ZipFile, is_zipfile, BadZipfile
  26
  27 from httplib import responses
  28 from requests import get, codes, RequestException, Timeout, TooManyRedirects, \
  29     HTTPError, ConnectionError
  30
  31 from errors import PresentationError
  32
  33
  34 # Chunk size used for file download
  35 CHUNK_SIZE = 512
  36
  37 # Separator used in file names
  38 SEPARATOR = "__"
  39
  40 REGEX_RELEASE = re.compile(r'(\D*)(\d{4}|master)(\D*)')
  41
  42
  43 def download_data_files(spec):
  44     """Download all data specified in the specification file in the section
  45     type: input --> builds.
  46
  47     :param spec: Specification.
  48     :type spec: Specification
  49     :raises: PresentationError if there is no url defined for the job.
  50     """
  51
  52     for job, builds in spec.builds.items():
  53         for build in builds:
  54             if job.startswith("csit-"):
  55                 if spec.input["file-name"].endswith(".zip"):
  56                     url = spec.environment["urls"]["URL[JENKINS,CSIT]"]
  57                 elif spec.input["file-name"].endswith(".gz"):
  58                     url = spec.environment["urls"]["URL[NEXUS,LOG]"]
  59                 else:
  60                     logging.error("Not supported file format.")
  61                     continue
  62             elif job.startswith("hc2vpp-"):
  63                 url = spec.environment["urls"]["URL[JENKINS,HC]"]
  64             else:
  65                 raise PresentationError("No url defined for the job '{}'.".
  66                                         format(job))
  67             file_name = spec.input["file-name"]
  68             full_name = spec.input["download-path"].\
  69                 format(job=job, build=build["build"], filename=file_name)
  70             url = "{0}/{1}".format(url, full_name)
  71             new_name = join(
  72                 spec.environment["paths"]["DIR[WORKING,DATA]"],
  73                 "{job}{sep}{build}{sep}{name}".format(job=job,
  74                                                       sep=SEPARATOR,
  75                                                       build=build["build"],
  76                                                       name=file_name))
  77             logging.info(
  78                 "Downloading the file '{0}' to '{1}' ...".format(url, new_name))
  79
  80             status = "failed"
  81             try:
  82                 response = get(url, stream=True)
  83                 code = response.status_code
  84                 # temporary workaround, remove when output.log.xml is not needed
  85                 if code != codes["OK"] and \
  86                         spec.input["file-name"].endswith(".gz"):
  87                     url = '.'.join(url.split('.')[:-1]) + ".log.gz"
  88                     response = get(url, stream=True)
  89                     code = response.status_code
  90
  91                 if code != codes["OK"]:
  92                     logging.warning(
  93                         "Jenkins: {0}: {1}.".format(code, responses[code]))
  94                     logging.info("Trying to download from Nexus:")
  95                     spec.set_input_state(job, build["build"], "not found")
  96                     if code == codes["not_found"]:
  97                         release = re.search(REGEX_RELEASE, job).group(2)
  98                         nexus_file_name = "{job}{sep}{build}{sep}{name}".\
  99                             format(job=job, sep=SEPARATOR, build=build["build"],
 100                                    name=file_name)
 101                         url = "{url}/rls{release}/{dir}/{file}".\
 102                             format(url=spec.environment["urls"]["URL[NEXUS]"],
 103                                    release=release,
 104                                    dir=spec.environment["urls"]["DIR[NEXUS]"],
 105                                    file=nexus_file_name)
 106                         logging.info("Downloading the file '{0}' to '{1}' ...".
 107                                      format(url, new_name))
 108                         response = get(url, stream=True)
 109                         code = response.status_code
 110                         if code != codes["OK"]:
 111                             logging.error(
 112                                 "Nexus: {0}: {1}".format(code, responses[code]))
 113                             spec.set_input_state(
 114                                 job, build["build"], "not found")
 115                             continue
 116
 117                 file_handle = open(new_name, "wb")
 118                 for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
 119                     if chunk:
 120                         file_handle.write(chunk)
 121                 file_handle.close()
 122
 123                 if spec.input["file-name"].endswith(".zip"):
 124                     expected_length = None
 125                     try:
 126                         expected_length = int(response.
 127                                               headers["Content-Length"])
 128                         logging.debug("  Expected file size: {0}B".
 129                                       format(expected_length))
 130                     except KeyError:
 131                         logging.debug("  No information about expected size.")
 132
 133                     real_length = getsize(new_name)
 134                     logging.debug("  Downloaded size: {0}B".format(real_length))
 135
 136                     if expected_length:
 137                         if real_length == expected_length:
 138                             status = "downloaded"
 139                             logging.info("{0}: {1}".format(code,
 140                                                            responses[code]))
 141                         else:
 142                             logging.error("The file size differs from the "
 143                                           "expected size.")
 144                     else:
 145                         status = "downloaded"
 146                         logging.info("{0}: {1}".format(code, responses[code]))
 147
 148                 elif spec.input["file-name"].endswith(".gz"):
 149                     rename(new_name, new_name[:-3])
 150                     with open(new_name[:-3], 'r') as xml_file:
 151                         with gzip.open(new_name, 'wb') as gz_file:
 152                             gz_file.write(xml_file.read())
 153                     new_name = new_name[:-3]
 154                     status = "downloaded"
 155                     logging.info("{0}: {1}".format(code, responses[code]))
 156
 157             except ConnectionError as err:
 158                 logging.error("Not possible to connect to '{0}'.".format(url))
 159                 logging.debug(err)
 160             except HTTPError as err:
 161                 logging.error("Invalid HTTP response from '{0}'.".format(url))
 162                 logging.debug(err)
 163             except TooManyRedirects as err:
 164                 logging.error("Request exceeded the configured number "
 165                               "of maximum re-directions.")
 166                 logging.debug(err)
 167             except Timeout as err:
 168                 logging.error("Request timed out.")
 169                 logging.debug(err)
 170             except RequestException as err:
 171                 logging.error("Unexpected HTTP request exception.")
 172                 logging.debug(err)
 173             except (IOError, ValueError, KeyError) as err:
 174                 logging.error("Download failed.")
 175                 logging.debug("Reason: {0}".format(err))
 176
 177             spec.set_input_state(job, build["build"], status)
 178             spec.set_input_file_name(job, build["build"], new_name)
 179
 180             if status == "failed":
 181                 logging.info("Removing the file '{0}'".format(new_name))
 182                 try:
 183                     remove(new_name)
 184                 except OSError as err:
 185                     logging.warning(str(err))
 186                 spec.set_input_file_name(job, build["build"], None)
 187
 188     unzip_files(spec)
 189
 190
 191 def unzip_files(spec):
 192     """Unzip downloaded zip files
 193
 194     :param spec: Specification.
 195     :type spec: Specification
 196     :raises: PresentationError if the zip file does not exist or it is not a
 197     zip file.
 198     """
 199
 200     if spec.is_debug:
 201         data_file = spec.debug["extract"]
 202     else:
 203         data_file = spec.input["extract"]
 204
 205     for job, builds in spec.builds.items():
 206         for build in builds:
 207             if build["status"] == "failed" or build["status"] == "not found":
 208                 continue
 209             try:
 210                 status = "failed"
 211                 directory = spec.environment["paths"]["DIR[WORKING,DATA]"]
 212                 file_name = join(build["file-name"])
 213
 214                 if build["status"] == "downloaded":
 215                     logging.info("Unziping: '{0}' from '{1}'.".
 216                                  format(data_file, file_name))
 217                     new_name = "{0}{1}{2}".format(file_name.rsplit('.')[-2],
 218                                                   SEPARATOR,
 219                                                   data_file.split("/")[-1])
 220                     try:
 221                         if is_zipfile(file_name):
 222                             with ZipFile(file_name, 'r') as zip_file:
 223                                 zip_file.extract(data_file, directory)
 224                             logging.info("Moving {0} to {1} ...".
 225                                          format(join(directory, data_file),
 226                                                 directory))
 227                             move(join(directory, data_file), directory)
 228                             logging.info("Renaming the file '{0}' to '{1}'".
 229                                          format(join(directory,
 230                                                      data_file.split("/")[-1]),
 231                                                 new_name))
 232                             rename(join(directory, data_file.split("/")[-1]),
 233                                    new_name)
 234                             spec.set_input_file_name(job, build["build"],
 235                                                      new_name)
 236                         status = "unzipped"
 237                         spec.set_input_state(job, build["build"], status)
 238                     except (BadZipfile, RuntimeError) as err:
 239                         logging.error("Failed to unzip the file '{0}': {1}.".
 240                                       format(file_name, str(err)))
 241                     except OSError as err:
 242                         logging.error("Failed to rename the file '{0}': {1}.".
 243                                       format(data_file, str(err)))
 244                     finally:
 245                         if status == "failed":
 246                             spec.set_input_file_name(job, build["build"], None)
 247                 else:
 248                     raise PresentationError("The file '{0}' does not exist or "
 249                                             "it is not a zip file".
 250                                             format(file_name))
 251
 252                 spec.set_input_state(job, build["build"], status)
 253
 254             except KeyError:
 255                 pass