resources/tools/presentation/input_data_files.py

   1 # Copyright (c) 2017 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Inputs
  15 Download all data.
  16 """
  17
  18 import logging
  19
  20 from os import rename, remove
  21 from os.path import join, getsize
  22 from shutil import move
  23 from zipfile import ZipFile, is_zipfile, BadZipfile
  24 from httplib import responses
  25 from requests import get, codes, RequestException, Timeout, TooManyRedirects, \
  26     HTTPError, ConnectionError
  27
  28 from errors import PresentationError
  29
  30
  31 # Chunk size used for file download
  32 CHUNK_SIZE = 512
  33
  34 # Separator used in file names
  35 SEPARATOR = "__"
  36
  37
  38 def download_data_files(spec):
  39     """Download all data specified in the specification file in the section
  40     type: input --> builds.
  41
  42     :param spec: Specification.
  43     :type spec: Specification
  44     :raises: PresentationError if there is no url defined for the job.
  45     """
  46
  47     for job, builds in spec.builds.items():
  48         for build in builds:
  49             if job.startswith("csit-"):
  50                 url = spec.environment["urls"]["URL[JENKINS,CSIT]"]
  51             elif job.startswith("hc2vpp-"):
  52                 url = spec.environment["urls"]["URL[JENKINS,HC]"]
  53             else:
  54                 raise PresentationError("No url defined for the job '{}'.".
  55                                         format(job))
  56             file_name = spec.input["file-name"]
  57             full_name = spec.input["download-path"].\
  58                 format(job=job, build=build["build"], filename=file_name)
  59             url = "{0}/{1}".format(url, full_name)
  60             new_name = join(
  61                 spec.environment["paths"]["DIR[WORKING,DATA]"],
  62                 "{job}{sep}{build}{sep}{name}".format(job=job, sep=SEPARATOR,
  63                                                       build=build["build"],
  64                                                       name=file_name))
  65
  66             logging.info("Downloading the file '{0}' to '{1}'.".
  67                          format(url, new_name))
  68
  69             status = "failed"
  70             try:
  71                 response = get(url, stream=True)
  72                 code = response.status_code
  73                 if code != codes["OK"]:
  74                     logging.error("{0}: {1}".format(code, responses[code]))
  75                     spec.set_input_state(job, build["build"], "not found")
  76                     continue
  77
  78                 file_handle = open(new_name, "wb")
  79                 for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
  80                     if chunk:
  81                         file_handle.write(chunk)
  82                 file_handle.close()
  83
  84                 expected_length = None
  85                 try:
  86                     expected_length = int(response.headers["Content-Length"])
  87                     logging.debug("  Expected file size: {0}B".
  88                                   format(expected_length))
  89                 except KeyError:
  90                     logging.debug("  No information about expected size.")
  91
  92                 real_length = getsize(new_name)
  93                 logging.debug("  Downloaded size: {0}B".format(real_length))
  94
  95                 if expected_length:
  96                     if real_length == expected_length:
  97                         status = "downloaded"
  98                         logging.info("{0}: {1}".format(code, responses[code]))
  99                     else:
 100                         logging.error("The file size differs from the expected "
 101                                       "size.")
 102                 else:
 103                     status = "downloaded"
 104                     logging.info("{0}: {1}".format(code, responses[code]))
 105
 106             except ConnectionError as err:
 107                 logging.error("Not possible to connect to '{0}'.".format(url))
 108                 logging.debug(err)
 109             except HTTPError as err:
 110                 logging.error("Invalid HTTP response from '{0}'.".format(url))
 111                 logging.debug(err)
 112             except TooManyRedirects as err:
 113                 logging.error("Request exceeded the configured number "
 114                               "of maximum re-directions.")
 115                 logging.debug(err)
 116             except Timeout as err:
 117                 logging.error("Request timed out.")
 118                 logging.debug(err)
 119             except RequestException as err:
 120                 logging.error("Unexpected HTTP request exception.")
 121                 logging.debug(err)
 122             except (IOError, ValueError, KeyError) as err:
 123                 logging.error("Download failed.")
 124                 logging.debug("Reason: {0}".format(err))
 125
 126             spec.set_input_state(job, build["build"], status)
 127             spec.set_input_file_name(job, build["build"], new_name)
 128
 129             if status == "failed":
 130                 logging.info("Removing the file '{0}'".format(new_name))
 131                 try:
 132                     remove(new_name)
 133                 except OSError as err:
 134                     logging.warning(str(err))
 135                 spec.set_input_file_name(job, build["build"], None)
 136
 137     unzip_files(spec)
 138
 139
 140 def unzip_files(spec):
 141     """Unzip downloaded zip files
 142
 143     :param spec: Specification.
 144     :type spec: Specification
 145     :raises: PresentationError if the zip file does not exist or it is not a
 146     zip file.
 147     """
 148
 149     if spec.is_debug:
 150         data_file = spec.debug["extract"]
 151     else:
 152         data_file = spec.input["extract"]
 153
 154     for job, builds in spec.builds.items():
 155         for build in builds:
 156             if build["status"] == "failed" or build["status"] == "not found":
 157                 continue
 158             try:
 159                 status = "failed"
 160                 directory = spec.environment["paths"]["DIR[WORKING,DATA]"]
 161                 file_name = join(build["file-name"])
 162
 163                 if build["status"] == "downloaded" and is_zipfile(file_name):
 164                     logging.info("Unziping: '{0}' from '{1}'.".
 165                                  format(data_file, file_name))
 166                     new_name = "{0}{1}{2}".format(file_name.rsplit('.')[-2],
 167                                                   SEPARATOR,
 168                                                   data_file.split("/")[-1])
 169                     try:
 170                         with ZipFile(file_name, 'r') as zip_file:
 171                             zip_file.extract(data_file, directory)
 172                         logging.info("Moving {0} to {1} ...".
 173                                      format(join(directory, data_file),
 174                                             directory))
 175                         move(join(directory, data_file), directory)
 176                         logging.info("Renaming the file '{0}' to '{1}'".
 177                                      format(join(directory,
 178                                                  data_file.split("/")[-1]),
 179                                             new_name))
 180                         rename(join(directory, data_file.split("/")[-1]),
 181                                new_name)
 182                         status = "unzipped"
 183                         spec.set_input_state(job, build["build"], status)
 184                         spec.set_input_file_name(job, build["build"],
 185                                                    new_name)
 186                     except (BadZipfile, RuntimeError) as err:
 187                         logging.error("Failed to unzip the file '{0}': {1}.".
 188                                       format(file_name, str(err)))
 189                     except OSError as err:
 190                         logging.error("Failed to rename the file '{0}': {1}.".
 191                                       format(data_file, str(err)))
 192                     finally:
 193                         if status == "failed":
 194                             spec.set_input_file_name(job, build["build"],
 195                                                        None)
 196                 else:
 197                     raise PresentationError("The file '{0}' does not exist or "
 198                                             "it is not a zip file".
 199                                             format(file_name))
 200
 201                 spec.set_input_state(job, build["build"], status)
 202
 203             except KeyError:
 204                 pass