resources/tools/presentation/inputs.py

   1 # Copyright (c) 2017 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Inputs
  15 Download all data.
  16 """
  17
  18 import logging
  19
  20 from os import rename, remove
  21 from os.path import join, getsize
  22 from zipfile import ZipFile, is_zipfile, BadZipfile
  23 from httplib import responses
  24 from requests import get, codes, RequestException, Timeout, TooManyRedirects, \
  25     HTTPError, ConnectionError
  26
  27 from errors import PresentationError
  28
  29
  30 # Chunk size used for file download
  31 CHUNK_SIZE = 512
  32
  33 # Separator used in file names
  34 SEPARATOR = "__"
  35
  36
  37 def download_data_files(config):
  38     """Download all data specified in the configuration file in the section
  39     type: input --> builds.
  40
  41     :param config: Configuration.
  42     :type config: Configuration
  43     :raises: PresentationError if there is no url defined for the job.
  44     """
  45
  46     for job, builds in config.builds.items():
  47         for build in builds:
  48             if job.startswith("csit-"):
  49                 url = config.environment["urls"]["URL[JENKINS,CSIT]"]
  50             elif job.startswith("hc2vpp-"):
  51                 url = config.environment["urls"]["URL[JENKINS,HC]"]
  52             else:
  53                 raise PresentationError("No url defined for the job '{}'.".
  54                                         format(job))
  55             file_name = config.input["file-name"]
  56             full_name = config.input["download-path"].\
  57                 format(job=job, build=build["build"], filename=file_name)
  58             url = "{0}/{1}".format(url, full_name)
  59             new_name = join(
  60                 config.environment["paths"]["DIR[WORKING,DATA]"],
  61                 "{job}{sep}{build}{sep}{name}".format(job=job, sep=SEPARATOR,
  62                                                       build=build["build"],
  63                                                       name=file_name))
  64
  65             logging.info("Downloading the file '{0}' to '{1}'.".
  66                          format(url, new_name))
  67
  68             status = "failed"
  69             try:
  70                 response = get(url, stream=True)
  71                 code = response.status_code
  72                 if code != codes["OK"]:
  73                     logging.error("{0}: {1}".format(code, responses[code]))
  74                     config.set_input_state(job, build["build"], "not found")
  75                     break
  76
  77                 file_handle = open(new_name, "wb")
  78                 for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
  79                     if chunk:
  80                         file_handle.write(chunk)
  81                 file_handle.close()
  82
  83                 expected_length = int(response.headers["Content-Length"])
  84                 logging.debug("  Expected file size: {0}B".
  85                               format(expected_length))
  86                 real_length = getsize(new_name)
  87                 logging.debug("  Downloaded size: {0}B".format(real_length))
  88
  89                 if real_length == expected_length:
  90                     status = "downloaded"
  91                     logging.info("{0}: {1}".format(code, responses[code]))
  92                 else:
  93                     logging.error("The file size differs from the expected "
  94                                   "size.")
  95             except ConnectionError as err:
  96                 logging.error("Not possible to connect to '{0}'.".format(url))
  97                 logging.debug(err)
  98             except HTTPError as err:
  99                 logging.error("Invalid HTTP response from '{0}'.".format(url))
 100                 logging.debug(err)
 101             except TooManyRedirects as err:
 102                 logging.error("Request exceeded the configured number "
 103                               "of maximum re-directions.")
 104                 logging.debug(err)
 105             except Timeout as err:
 106                 logging.error("Request timed out.")
 107                 logging.debug(err)
 108             except RequestException as err:
 109                 logging.error("Unexpected HTTP request exception.")
 110                 logging.debug(err)
 111             except (IOError, ValueError, KeyError) as err:
 112                 logging.error("Download failed.")
 113                 logging.debug("Reason: {0}".format(err))
 114
 115             config.set_input_state(job, build["build"], status)
 116             config.set_input_file_name(job, build["build"], new_name)
 117
 118             if status == "failed":
 119                 logging.info("Removing the file '{0}'".format(new_name))
 120                 try:
 121                     remove(new_name)
 122                 except OSError as err:
 123                     logging.warning(str(err))
 124                 config.set_input_file_name(job, build["build"], None)
 125
 126     unzip_files(config)
 127
 128
 129 def unzip_files(config):
 130     """Unzip downloaded zip files
 131
 132     :param config: Configuration.
 133     :type config: Configuration
 134     :raises: PresentationError if the zip file does not exist or it is not a
 135     zip file.
 136     """
 137
 138     if config.is_debug:
 139         data_file = config.debug["extract"]
 140     else:
 141         data_file = config.input["extract"]
 142
 143     for job, builds in config.builds.items():
 144         for build in builds:
 145             try:
 146                 status = "failed"
 147                 file_name = build["file-name"]
 148                 directory = config.environment["paths"]["DIR[WORKING,DATA]"]
 149                 if build["status"] == "downloaded" and is_zipfile(file_name):
 150                     logging.info("Unziping: '{0}' from '{1}'.".
 151                                  format(data_file, file_name))
 152                     new_name = "{0}{1}{2}".format(file_name.rsplit('.')[-2],
 153                                                   SEPARATOR, data_file)
 154                     try:
 155                         with ZipFile(file_name, 'r') as zip_file:
 156                             zip_file.extract(data_file, directory)
 157                         logging.info("Renaming the file '{0}' to '{1}'".
 158                                      format(data_file, new_name))
 159                         rename(join(directory, data_file), new_name)
 160                         status = "unzipped"
 161                         config.set_input_state(job, build["build"], status)
 162                         config.set_input_file_name(job, build["build"],
 163                                                    new_name)
 164                     except (BadZipfile, RuntimeError) as err:
 165                         logging.error("Failed to unzip the file '{0}': {1}.".
 166                                       format(file_name, str(err)))
 167                     except OSError as err:
 168                         logging.error("Failed to rename the file '{0}': {1}.".
 169                                       format(data_file, str(err)))
 170                     finally:
 171                         logging.info("Removing the file '{0}'".
 172                                      format(file_name))
 173                         try:
 174                             if not config.debug:
 175                                 remove(file_name)
 176                         except OSError as err:
 177                             logging.warning(str(err))
 178                         if status == "failed":
 179                             config.set_input_file_name(job, build["build"],
 180                                                        None)
 181                 else:
 182                     raise PresentationError("The file '{0}' does not exist or "
 183                                             "it is not a zip file".
 184                                             format(file_name))
 185
 186                 config.set_input_state(job, build["build"], status)
 187
 188             except KeyError:
 189                 pass