resources/tools/presentation/input_data_files.py

   1 # Copyright (c) 2017 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Inputs
  15 Download all data.
  16 """
  17
  18 import re
  19
  20 import logging
  21
  22 from os import rename, remove
  23 from os.path import join, getsize
  24 from shutil import move
  25 from zipfile import ZipFile, is_zipfile, BadZipfile
  26 from httplib import responses
  27 from requests import get, codes, RequestException, Timeout, TooManyRedirects, \
  28     HTTPError, ConnectionError
  29
  30 from errors import PresentationError
  31
  32
  33 # Chunk size used for file download
  34 CHUNK_SIZE = 512
  35
  36 # Separator used in file names
  37 SEPARATOR = "__"
  38
  39 REGEX_RELEASE = re.compile(r'(\D*)(\d{4})(\D*)')
  40
  41
  42 def download_data_files(spec):
  43     """Download all data specified in the specification file in the section
  44     type: input --> builds.
  45
  46     :param spec: Specification.
  47     :type spec: Specification
  48     :raises: PresentationError if there is no url defined for the job.
  49     """
  50
  51     for job, builds in spec.builds.items():
  52         for build in builds:
  53             if job.startswith("csit-"):
  54                 url = spec.environment["urls"]["URL[JENKINS,CSIT]"]
  55             elif job.startswith("hc2vpp-"):
  56                 url = spec.environment["urls"]["URL[JENKINS,HC]"]
  57             else:
  58                 raise PresentationError("No url defined for the job '{}'.".
  59                                         format(job))
  60             file_name = spec.input["file-name"]
  61             full_name = spec.input["download-path"].\
  62                 format(job=job, build=build["build"], filename=file_name)
  63             url = "{0}/{1}".format(url, full_name)
  64             new_name = join(
  65                 spec.environment["paths"]["DIR[WORKING,DATA]"],
  66                 "{job}{sep}{build}{sep}{name}".format(job=job,
  67                                                       sep=SEPARATOR,
  68                                                       build=build["build"],
  69                                                       name=file_name))
  70             logging.info(
  71                 "Downloading the file '{0}' to '{1}' ...".format(url, new_name))
  72
  73             status = "failed"
  74             try:
  75                 response = get(url, stream=True)
  76                 code = response.status_code
  77                 if code != codes["OK"]:
  78                     logging.warning(
  79                         "Jenkins: {0}: {1}.".format(code, responses[code]))
  80                     logging.info("Trying to download from Nexus:")
  81                     spec.set_input_state(job, build["build"], "not found")
  82                     if code == codes["not_found"]:
  83                         release = re.search(REGEX_RELEASE, job).group(2)
  84                         nexus_file_name = "{job}{sep}{build}{sep}{name}".\
  85                             format(job=job, sep=SEPARATOR, build=build["build"],
  86                                    name=file_name)
  87                         url = "{url}/rls{release}/{dir}/{file}".\
  88                             format(url=spec.environment["urls"]["URL[NEXUS]"],
  89                                    release=release,
  90                                    dir=spec.environment["urls"]["DIR[NEXUS]"],
  91                                    file=nexus_file_name)
  92                         logging.info("Downloading the file '{0}' to '{1}' ...".
  93                                      format(url, new_name))
  94                         response = get(url, stream=True)
  95                         code = response.status_code
  96                         if code != codes["OK"]:
  97                             logging.error(
  98                                 "Nexus: {0}: {1}".format(code, responses[code]))
  99                             spec.set_input_state(
 100                                 job, build["build"], "not found")
 101                             continue
 102
 103                 file_handle = open(new_name, "wb")
 104                 for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
 105                     if chunk:
 106                         file_handle.write(chunk)
 107                 file_handle.close()
 108
 109                 expected_length = None
 110                 try:
 111                     expected_length = int(response.headers["Content-Length"])
 112                     logging.debug("  Expected file size: {0}B".
 113                                   format(expected_length))
 114                 except KeyError:
 115                     logging.debug("  No information about expected size.")
 116
 117                 real_length = getsize(new_name)
 118                 logging.debug("  Downloaded size: {0}B".format(real_length))
 119
 120                 if expected_length:
 121                     if real_length == expected_length:
 122                         status = "downloaded"
 123                         logging.info("{0}: {1}".format(code, responses[code]))
 124                     else:
 125                         logging.error("The file size differs from the expected "
 126                                       "size.")
 127                 else:
 128                     status = "downloaded"
 129                     logging.info("{0}: {1}".format(code, responses[code]))
 130
 131             except ConnectionError as err:
 132                 logging.error("Not possible to connect to '{0}'.".format(url))
 133                 logging.debug(err)
 134             except HTTPError as err:
 135                 logging.error("Invalid HTTP response from '{0}'.".format(url))
 136                 logging.debug(err)
 137             except TooManyRedirects as err:
 138                 logging.error("Request exceeded the configured number "
 139                               "of maximum re-directions.")
 140                 logging.debug(err)
 141             except Timeout as err:
 142                 logging.error("Request timed out.")
 143                 logging.debug(err)
 144             except RequestException as err:
 145                 logging.error("Unexpected HTTP request exception.")
 146                 logging.debug(err)
 147             except (IOError, ValueError, KeyError) as err:
 148                 logging.error("Download failed.")
 149                 logging.debug("Reason: {0}".format(err))
 150
 151             spec.set_input_state(job, build["build"], status)
 152             spec.set_input_file_name(job, build["build"], new_name)
 153
 154             if status == "failed":
 155                 logging.info("Removing the file '{0}'".format(new_name))
 156                 try:
 157                     remove(new_name)
 158                 except OSError as err:
 159                     logging.warning(str(err))
 160                 spec.set_input_file_name(job, build["build"], None)
 161
 162     unzip_files(spec)
 163
 164
 165 def unzip_files(spec):
 166     """Unzip downloaded zip files
 167
 168     :param spec: Specification.
 169     :type spec: Specification
 170     :raises: PresentationError if the zip file does not exist or it is not a
 171     zip file.
 172     """
 173
 174     if spec.is_debug:
 175         data_file = spec.debug["extract"]
 176     else:
 177         data_file = spec.input["extract"]
 178
 179     for job, builds in spec.builds.items():
 180         for build in builds:
 181             if build["status"] == "failed" or build["status"] == "not found":
 182                 continue
 183             try:
 184                 status = "failed"
 185                 directory = spec.environment["paths"]["DIR[WORKING,DATA]"]
 186                 file_name = join(build["file-name"])
 187
 188                 if build["status"] == "downloaded" and is_zipfile(file_name):
 189                     logging.info("Unziping: '{0}' from '{1}'.".
 190                                  format(data_file, file_name))
 191                     new_name = "{0}{1}{2}".format(file_name.rsplit('.')[-2],
 192                                                   SEPARATOR,
 193                                                   data_file.split("/")[-1])
 194                     try:
 195                         with ZipFile(file_name, 'r') as zip_file:
 196                             zip_file.extract(data_file, directory)
 197                         logging.info("Moving {0} to {1} ...".
 198                                      format(join(directory, data_file),
 199                                             directory))
 200                         move(join(directory, data_file), directory)
 201                         logging.info("Renaming the file '{0}' to '{1}'".
 202                                      format(join(directory,
 203                                                  data_file.split("/")[-1]),
 204                                             new_name))
 205                         rename(join(directory, data_file.split("/")[-1]),
 206                                new_name)
 207                         status = "unzipped"
 208                         spec.set_input_state(job, build["build"], status)
 209                         spec.set_input_file_name(job, build["build"],
 210                                                    new_name)
 211                     except (BadZipfile, RuntimeError) as err:
 212                         logging.error("Failed to unzip the file '{0}': {1}.".
 213                                       format(file_name, str(err)))
 214                     except OSError as err:
 215                         logging.error("Failed to rename the file '{0}': {1}.".
 216                                       format(data_file, str(err)))
 217                     finally:
 218                         if status == "failed":
 219                             spec.set_input_file_name(job, build["build"],
 220                                                        None)
 221                 else:
 222                     raise PresentationError("The file '{0}' does not exist or "
 223                                             "it is not a zip file".
 224                                             format(file_name))
 225
 226                 spec.set_input_state(job, build["build"], status)
 227
 228             except KeyError:
 229                 pass